# sudo -u hive hive --hiveconf hive.metastore.uris=thrift://10.197.1.141:9084hive> create database wgtestdb_region3_1;OKTime taken: 200.826 seconds

# sudo -u hive hive --hiveconf hive.metastore.uris=thrift://10.197.1.160:9084hive> create database wgtestdb_region2_2;OKTime taken: 0.112 seconds

2021-02-08 16:54:17,910 ERROR org.apache.sentry.core.common.transport.RetryClientInvocationHandler: [pool-5-thread-670]: failed to execute syncNotificationsjava.lang.reflect.InvocationTargetException        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)        at java.lang.reflect.Method.invoke(Method.java:498)        at org.apache.sentry.core.common.transport.RetryClientInvocationHandler.invokeImpl(RetryClientInvocationHandler.java:95)        at org.apache.sentry.core.common.transport.SentryClientInvocationHandler.invoke(SentryClientInvocationHandler.java:41)        at com.sun.proxy.$Proxy26.syncNotifications(Unknown Source)        at org.apache.sentry.binding.metastore.SentrySyncHMSNotificationsPostEventListener.syncNotificationEvents(SentrySyncHMSNotificationsPostEventListener.java:153)        at org.apache.sentry.binding.metastore.SentrySyncHMSNotificationsPostEventListener.onDropDatabase(SentrySyncHMSNotificationsPostEventListener.java:113)        at org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier$13.notify(MetaStoreListenerNotifier.java:69)        at org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:167)        at org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:197)        at org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier.notifyEvent(MetaStoreListenerNotifier.java:235)        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.drop_database_core(HiveMetaStore.java:1193)        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.drop_database(HiveMetaStore.java:1229)        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)        at java.lang.reflect.Method.invoke(Method.java:498)        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:140)        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:99)        at com.sun.proxy.$Proxy11.drop_database(Unknown Source)        at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$drop_database.getResult(ThriftHiveMetastore.java:9624)        at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$drop_database.getResult(ThriftHiveMetastore.java:9608)        at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)        at org.apache.hadoop.hive.metastore.TUGIBasedProcessor$1.run(TUGIBasedProcessor.java:110)        at org.apache.hadoop.hive.metastore.TUGIBasedProcessor$1.run(TUGIBasedProcessor.java:106)        at java.security.AccessController.doPrivileged(Native Method)        at javax.security.auth.Subject.doAs(Subject.java:422)        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924)        at org.apache.hadoop.hive.metastore.TUGIBasedProcessor.process(TUGIBasedProcessor.java:118)        at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)        at java.lang.Thread.run(Thread.java:748)Caused by: java.lang.RuntimeException: timed out wait request for id 184474148. Server Stacktrace: java.util.concurrent.TimeoutException        at org.apache.sentry.service.thrift.CounterWait$ValueEvent.waitFor(CounterWait.java:299)        at org.apache.sentry.service.thrift.CounterWait.waitFor(CounterWait.java:212)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor.sentry_sync_notifications(SentryPolicyStoreProcessor.java:934)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyService$Processor$sentry_sync_notifications.getResult(SentryPolicyService.java:1217)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyService$Processor$sentry_sync_notifications.getResult(SentryPolicyService.java:1202)        at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)        at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)        at org.apache.sentry.provider.db.service.thrift.SentryProcessorWrapper.process(SentryProcessorWrapper.java:36)        at org.apache.thrift.TMultiplexedProcessor.process(TMultiplexedProcessor.java:123)        at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)        at java.lang.Thread.run(Thread.java:748)
        at org.apache.sentry.service.thrift.Status.throwIfNotOk(Status.java:109)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClientDefaultImpl.syncNotifications(SentryPolicyServiceClientDefaultImpl.java:824)        ... 35 more

2021-02-08 15:29:09,028 WARN org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor: timed out wait request for id 184444094java.util.concurrent.TimeoutException        at org.apache.sentry.service.thrift.CounterWait$ValueEvent.waitFor(CounterWait.java:299)        at org.apache.sentry.service.thrift.CounterWait.waitFor(CounterWait.java:212)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor.sentry_sync_notifications(SentryPolicyStoreProcessor.java:934)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyService$Processor$sentry_sync_notifications.getResult(SentryPolicyService.java:1217)        at org.apache.sentry.provider.db.service.thrift.SentryPolicyService$Processor$sentry_sync_notifications.getResult(SentryPolicyService.java:1202)        at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)        at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)        at org.apache.sentry.provider.db.service.thrift.SentryProcessorWrapper.process(SentryProcessorWrapper.java:36)        at org.apache.thrift.TMultiplexedProcessor.process(TMultiplexedProcessor.java:123)        at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)        at java.lang.Thread.run(Thread.java:748)

# sentry 元数据信息mysql -uroot -p123456use sentry;mysql> select * from  SENTRY_HMS_NOTIFICATION_ID order by NOTIFICATION_ID desc limit 5;（一段时间不更新，此时 HMS canary 异常）+-----------------+| NOTIFICATION_ID |+-----------------+|       184485024 ||       184485023 ||       184485023 ||       184485022 ||       184485021 |+-----------------+5 rows in set (0.00 sec)
# hive 元数据信息use hive_warehouse;MySQL [hive_warehouse]>  select * from NOTIFICATION_SEQUENCE limit 10;（持续更新）+--------+---------------+| NNI_ID | NEXT_EVENT_ID |+--------+---------------+|      1 |     184486656 |+--------+---------------+1 row in set (0.00 sec)

//位置：org/apache/hadoop/hive/metastore/HiveMetaStore.java    private void create_table_core(final RawStore ms, final Table tbl,        final EnvironmentContext envContext)        throws AlreadyExistsException, MetaException,        InvalidObjectException, NoSuchObjectException {
      ...        if (!transactionalListeners.isEmpty()) {          transactionalListenerResponses =              MetaStoreListenerNotifier.notifyEvent(transactionalListeners,                                                    EventType.CREATE_TABLE,                                                    new CreateTableEvent(tbl, true, this),                                                    envContext);        }      ...    }
    @Override    public void create_table(final Table tbl) throws AlreadyExistsException,        MetaException, InvalidObjectException {      create_table_with_environment_context(tbl, null);    }

//位置：org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java  /**   * Notify a list of listeners about a specific metastore event. Each listener notified might update   * the (ListenerEvent) event by setting a parameter key/value pair. These updated parameters will   * be returned to the caller.   *   * @param listeners List of MetaStoreEventListener listeners.   * @param eventType Type of the notification event.   * @param event The ListenerEvent with information about the event.   * @return A list of key/value pair parameters that the listeners set. The returned object will return an empty   *         map if no parameters were updated or if no listeners were notified.   * @throws MetaException If an error occurred while calling the listeners.   */  public static Map<String, String> notifyEvent(List<MetaStoreEventListener> listeners,                                                EventType eventType,                                                ListenerEvent event) throws MetaException {
    Preconditions.checkNotNull(listeners, "Listeners must not be null.");    Preconditions.checkNotNull(event, "The event must not be null.");
    for (MetaStoreEventListener listener : listeners) {      notificationEvents.get(eventType).notify(listener, event);    }
    // Each listener called above might set a different parameter on the event.    // This write permission is allowed on the listener side to avoid breaking compatibility if we change the API    // method calls.    return event.getParameters();  }

//位置：org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java/** * This HMS post-event listener is used only to synchronize with HMS notifications on the Sentry server * whenever a DDL event happens on the Hive metastore. */public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventListener {    /**   * It requests the Sentry server the synchronization of recent notification events.   *   * After the sync call, the latest processed ID will be stored for future reference to avoid   * syncing an ID that was already processed.   *   * @param event An event that contains a DB_NOTIFICATION_EVENT_ID_KEY_NAME value to request.   */  private void syncNotificationEvents(ListenerEvent event, String eventName) {    // Do not sync notifications if the event has failed.    if (failedEvent(event, eventName)) {      return;    }
    Map<String, String> eventParameters = event.getParameters();    if (!eventParameters.containsKey(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) {      return;    }
    /* If the HMS is running in an active transaction, then we do not want to sync with Sentry     * because the desired eventId is not available for Sentry yet, and Sentry may block the HMS     * forever or until a read time-out happens. */    if (isMetastoreTransactionActive(eventParameters)) {      return;    }
    long eventId =        Long.parseLong(eventParameters.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME));
    // This check is only for performance reasons to avoid calling the sync thrift call if the Sentry server    // already processed the requested eventId.    if (eventId <= latestProcessedId.get()) {      return;    }
    try(SentryPolicyServiceClient sentryClient = this.getSentryServiceClient()) {      LOGGER.debug("Starting Sentry/HMS notifications sync for {} (id: {})", eventName, eventId);      long sentryLatestProcessedId = sentryClient.syncNotifications(eventId);      LOGGER.debug("Finishedd Sentry/HMS notifications sync for {} (id: {})", eventName, eventId);      LOGGER.debug("Latest processed event ID returned by the Sentry server: {}", sentryLatestProcessedId);
      updateProcessedId(sentryLatestProcessedId);    } catch (Exception e) {      // This error is only logged. There is no need to throw an error to Hive because HMS sync is called      // after the notification is already generated by Hive (as post-event).      LOGGER.error("Failed to sync requested HMS notifications up to the event ID: " + eventId, e);    }  }}

//位置：org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java  @Override  public TSentrySyncIDResponse sentry_sync_notifications(TSentrySyncIDRequest request)          throws TException {    TSentrySyncIDResponse response = new TSentrySyncIDResponse();    try (Timer.Context timerContext = hmsWaitTimer.time()) {      // Wait until Sentry Server processes specified HMS Notification ID.      response.setId(sentryStore.getCounterWait().waitFor(request.getId()));      response.setStatus(Status.OK());    } catch (InterruptedException e) {      String msg = String.format("wait request for id %d is interrupted",              request.getId());      LOGGER.error(msg, e);      response.setId(0);      response.setStatus(Status.RuntimeError(msg, e));      Thread.currentThread().interrupt();    } catch (TimeoutException e) {      String msg = String.format("timed out wait request for id %d", request.getId());      LOGGER.warn(msg, e);      response.setId(0);      response.setStatus(Status.RuntimeError(msg, e));    }    return response;  }

//位置：org/apache/sentry/service/thrift/CounterWait.java  /**   * Wait for specified counter value.   * Returns immediately if the value is reached or blocks until the value   * is reached.   * Multiple threads can call the method concurrently.   *   * @param value requested counter value   * @return current counter value that should be no smaller then the requested   * value   * @throws InterruptedException if the wait was interrupted, TimeoutException if   * wait was not successfull within the timeout value specified at the construction time.   */  public long waitFor(long value) throws InterruptedException, TimeoutException {    // Fast path - counter value already reached, no need to block    if (value <= currentId.get()) {      return currentId.get();    }
    // Enqueue the waiter for this value    ValueEvent eid = new ValueEvent(value);    waiters.put(eid);
    // It is possible that between the fast path check and the time the    // value event is enqueued, the counter value already reached the requested    // value. In this case we return immediately.    if (value <= currentId.get()) {      return currentId.get();    }
    // At this point we may be sure that by the time the event was enqueued,    // the counter was below the requested value. This means that update()    // is guaranteed to wake us up when the counter reaches the requested value.    // The wake up may actually happen before we start waiting, in this case    // the event's blocking queue will be non-empty and the waitFor() below    // will not block, so it is safe to wake up before the wait.    // So sit tight and wait patiently.    eid.waitFor();    LOGGER.debug("CounterWait added new value to waitFor: value = {}, currentId = {}", value, currentId.get());    return currentId.get();  }

mysql>  select * from  SENTRY_HMS_NOTIFICATION_ID order by NOTIFICATION_ID desc limit 1;+-----------------+| NOTIFICATION_ID |+-----------------+|       184490926 |+-----------------+1 row in set (0.00 sec)

use sentry;insert into SENTRY_HMS_NOTIFICATION_ID values(184472866);更新后 create 操作时间正常