отчет об ошибке Не удается обновить объект с помощью версии mongoDB

Я тестирую производительность kaa Server. Когда я запускаю 50 тысяч клиентов, загружаю журнал каждые 10 секунд для каждого клиента.

2016-11-18 09:06:22,235 [EPS-endpoint-dispatcher-50] ERROR o.k.k.s.c.n.m.d.AbstractVersionableMongoDao - [class org.kaaproject.kaa.server.common.nosql.mongo.dao.model.MongoEndpointUser] Can't update entity with version 50730. Entity already changed!
2016-11-18 09:06:22,236 [EPS-core-dispatcher-3949] ERROR o.k.k.s.o.s.a.a.s.SupervisionStrategyFactory - Supervisor strategy got exception: Can't update entity with version 50730. Entity already changed!
org.kaaproject.kaa.server.common.dao.exception.KaaOptimisticLockingFailureException: Can't update entity with version 50730. Entity already changed!
        at org.kaaproject.kaa.server.common.nosql.mongo.dao.AbstractVersionableMongoDao.save(AbstractVersionableMongoDao.java:46) ~[mongo-dao-0.10.0.jar:na]
        at org.kaaproject.kaa.server.common.nosql.mongo.dao.AbstractVersionableMongoDao.save(AbstractVersionableMongoDao.java:26) ~[mongo-dao-0.10.0.jar:na]
        at sun.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[na:na]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_111]
        at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:302) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136) ~[spring-tx-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:208) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at com.sun.proxy.$Proxy76.save(Unknown Source) ~[na:na]
        at org.kaaproject.kaa.server.common.dao.service.EndpointServiceImpl.attachEndpointToUser(EndpointServiceImpl.java:416) ~[dao-0.10.0.jar:na]
        at sun.reflect.GeneratedMethodAccessor93.invoke(Unknown Source) ~[na:na]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_111]
        at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:302) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.retry.annotation.AnnotationAwareRetryOperationsInterceptor.invoke(AnnotationAwareRetryOperationsInterceptor.java:121) ~[spring-retry-1.1.2.RELEASE.jar:na]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:208) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at com.sun.proxy.$Proxy74.attachEndpointToUser(Unknown Source) ~[na:na]
        at org.kaaproject.kaa.server.operations.service.user.DefaultEndpointUserService.attachEndpointToUser(DefaultEndpointUserService.java:93) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.DefaultOperationsService.attachEndpointToUser(DefaultOperationsService.java:788) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.akka.actors.core.endpoint.local.LocalEndpointActorMessageProcessor.processUserVerificationMessage(LocalEndpointActorMessageProcessor.java:1025) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.akka.actors.core.endpoint.local.LocalEndpointActor.processUserVerificationMessage(LocalEndpointActor.java:123) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.akka.actors.core.endpoint.local.LocalEndpointActor.onReceive(LocalEndpointActor.java:88) ~[kaa-node-0.10.0.jar:na]
        at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.Actor$class.aroundReceive(Actor.scala:480) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.ActorCell.invoke(ActorCell.scala:495) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257) [akka-actor_2.11-2.4.1.jar:na]
        at akka.dispatch.Mailbox.run(Mailbox.scala:224) [akka-actor_2.11-2.4.1.jar:na]
        at akka.dispatch.Mailbox.exec(Mailbox.scala:234) [akka-actor_2.11-2.4.1.jar:na]
        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [scala-library-2.11.7.jar:na]
        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [scala-library-2.11.7.jar:na]
        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [scala-library-2.11.7.jar:na]
        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [scala-library-2.11.7.jar:na]
2016-11-18 09:06:22,237 [EPS-akka.actor.default-dispatcher-3469] ERROR akka.actor.OneForOneStrategy - Can't update entity with version 50730. Entity already changed!
org.kaaproject.kaa.server.common.dao.exception.KaaOptimisticLockingFailureException: Can't update entity with version 50730. Entity already changed!
        at org.kaaproject.kaa.server.common.nosql.mongo.dao.AbstractVersionableMongoDao.save(AbstractVersionableMongoDao.java:46) ~[mongo-dao-0.10.0.jar:na]
        at org.kaaproject.kaa.server.common.nosql.mongo.dao.AbstractVersionableMongoDao.save(AbstractVersionableMongoDao.java:26) ~[mongo-dao-0.10.0.jar:na]
        at sun.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[na:na]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_111]
        at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:302) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136) ~[spring-tx-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:208) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at com.sun.proxy.$Proxy76.save(Unknown Source) ~[na:na]
        at org.kaaproject.kaa.server.common.dao.service.EndpointServiceImpl.attachEndpointToUser(EndpointServiceImpl.java:416) ~[dao-0.10.0.jar:na]
        at sun.reflect.GeneratedMethodAccessor93.invoke(Unknown Source) ~[na:na]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_111]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_111]
        at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:302) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.retry.annotation.AnnotationAwareRetryOperationsInterceptor.invoke(AnnotationAwareRetryOperationsInterceptor.java:121) ~[spring-retry-1.1.2.RELEASE.jar:na]
        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:208) ~[spring-aop-4.2.5.RELEASE.jar:4.2.5.RELEASE]
        at com.sun.proxy.$Proxy74.attachEndpointToUser(Unknown Source) ~[na:na]
        at org.kaaproject.kaa.server.operations.service.user.DefaultEndpointUserService.attachEndpointToUser(DefaultEndpointUserService.java:93) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.DefaultOperationsService.attachEndpointToUser(DefaultOperationsService.java:788) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.akka.actors.core.endpoint.local.LocalEndpointActorMessageProcessor.processUserVerificationMessage(LocalEndpointActorMessageProcessor.java:1025) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.akka.actors.core.endpoint.local.LocalEndpointActor.processUserVerificationMessage(LocalEndpointActor.java:123) ~[kaa-node-0.10.0.jar:na]
        at org.kaaproject.kaa.server.operations.service.akka.actors.core.endpoint.local.LocalEndpointActor.onReceive(LocalEndpointActor.java:88) ~[kaa-node-0.10.0.jar:na]
        at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.Actor$class.aroundReceive(Actor.scala:480) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97) ~[akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526) [akka-actor_2.11-2.4.1.jar:na]
        at akka.actor.ActorCell.invoke(ActorCell.scala:495) [akka-actor_2.11-2.4.1.jar:na]
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257) [akka-actor_2.11-2.4.1.jar:na]
        at akka.dispatch.Mailbox.run(Mailbox.scala:224) [akka-actor_2.11-2.4.1.jar:na]
        at akka.dispatch.Mailbox.exec(Mailbox.scala:234) [akka-actor_2.11-2.4.1.jar:na]
        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [scala-library-2.11.7.jar:na]
        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [scala-library-2.11.7.jar:na]
        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [scala-library-2.11.7.jar:na]
        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [scala-library-2.11.7.jar:na]

Я обнаружил, что ОШИБКА вызвана AttachEndpointToUser. если большое количество клиентов сначала подключаются к пользователю, а у пользователя уже есть много конечных точек, выполнение метода AttachEndpointToUser в среднем потратит 5 секунд. Поэтому, возможно, многие клиенты обрабатывают одни и те же данные одновременно. Это приведет к тому, что endpoint_ids таблицы endpoint_user может иногда не обновлять версию Optimistic Lock.