diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java index 5a6d7a256e4..3528e048176 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java @@ -49,19 +49,15 @@ public void handlePutEvent(OMDBUpdateEvent event, HashMap unReplicatedSizeMap, HashMap replicatedSizeMap) { - String countKey = getTableCountKeyFromTable(tableName); - String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName); - String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName); - if (event.getValue() != null) { RepeatedOmKeyInfo repeatedOmKeyInfo = (RepeatedOmKeyInfo) event.getValue(); - objectCountMap.computeIfPresent(countKey, + objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count + repeatedOmKeyInfo.getOmKeyInfoList().size()); Pair result = repeatedOmKeyInfo.getTotalSize(); - unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey, + unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size + result.getLeft()); - replicatedSizeMap.computeIfPresent(replicatedSizeKey, + replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), (k, size) -> size + result.getRight()); } else { LOG.warn("Put event does not have the Key Info for {}.", @@ -81,19 +77,15 @@ public void handleDeleteEvent(OMDBUpdateEvent event, HashMap unReplicatedSizeMap, HashMap replicatedSizeMap) { - String countKey = getTableCountKeyFromTable(tableName); - String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName); - String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName); - if (event.getValue() != null) { RepeatedOmKeyInfo repeatedOmKeyInfo = (RepeatedOmKeyInfo) event.getValue(); - objectCountMap.computeIfPresent(countKey, (k, count) -> + objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count > 0 ? count - repeatedOmKeyInfo.getOmKeyInfoList().size() : 0L); Pair result = repeatedOmKeyInfo.getTotalSize(); - unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey, + unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size > result.getLeft() ? size - result.getLeft() : 0L); - replicatedSizeMap.computeIfPresent(replicatedSizeKey, + replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), (k, size) -> size > result.getRight() ? size - result.getRight() : 0L); } else { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java index 37a0e16e934..ced97a9b162 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java @@ -62,6 +62,13 @@ public class OmTableInsightTask implements ReconOmTask { private Configuration sqlConfiguration; private ReconOMMetadataManager reconOMMetadataManager; private Map tableHandlers; + private Collection tables; + private HashMap objectCountMap; + private HashMap unReplicatedSizeMap; + private HashMap replicatedSizeMap; + + private List insertGlobalStats; + private List updateGlobalStats; @Inject public OmTableInsightTask(GlobalStatsDao globalStatsDao, @@ -76,6 +83,9 @@ public OmTableInsightTask(GlobalStatsDao globalStatsDao, tableHandlers.put(OPEN_KEY_TABLE, new OpenKeysInsightHandler()); tableHandlers.put(OPEN_FILE_TABLE, new OpenKeysInsightHandler()); tableHandlers.put(DELETED_TABLE, new DeletedKeysInsightHandler()); + + insertGlobalStats = new ArrayList<>(); + updateGlobalStats = new ArrayList<>(); } /** @@ -92,16 +102,15 @@ public OmTableInsightTask(GlobalStatsDao globalStatsDao, */ @Override public Pair reprocess(OMMetadataManager omMetadataManager) { - HashMap objectCountMap = initializeCountMap(); - HashMap unReplicatedSizeMap = initializeSizeMap(false); - HashMap replicatedSizeMap = initializeSizeMap(true); + tables = getTaskTables(); + + // Initialize maps to store count and size information + objectCountMap = initializeCountMap(); + unReplicatedSizeMap = initializeSizeMap(false); + replicatedSizeMap = initializeSizeMap(true); - for (String tableName : getTaskTables()) { + for (String tableName : tables) { Table table = omMetadataManager.getTable(tableName); - if (table == null) { - LOG.error("Table " + tableName + " not found in OM Metadata."); - return new ImmutablePair<>(getTaskName(), false); - } try (TableIterator> iterator = table.iterator()) { @@ -157,35 +166,32 @@ public Collection getTaskTables() { @Override public Pair process(OMUpdateEventBatch events) { Iterator eventIterator = events.getIterator(); - // Initialize maps to store count and size information - HashMap objectCountMap = initializeCountMap(); - HashMap unReplicatedSizeMap = initializeSizeMap(false); - HashMap replicatedSizeMap = initializeSizeMap(true); - final Collection taskTables = getTaskTables(); + String tableName; + OMDBUpdateEvent omdbUpdateEvent; // Process each update event long startTime = System.currentTimeMillis(); while (eventIterator.hasNext()) { - OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - String tableName = omdbUpdateEvent.getTable(); - if (!taskTables.contains(tableName)) { + omdbUpdateEvent = eventIterator.next(); + tableName = omdbUpdateEvent.getTable(); + if (!tables.contains(tableName)) { continue; } try { switch (omdbUpdateEvent.getAction()) { case PUT: - handlePutEvent(omdbUpdateEvent, tableName, objectCountMap, - unReplicatedSizeMap, replicatedSizeMap); + handlePutEvent(omdbUpdateEvent, tableName + ); break; case DELETE: - handleDeleteEvent(omdbUpdateEvent, tableName, objectCountMap, - unReplicatedSizeMap, replicatedSizeMap); + handleDeleteEvent(omdbUpdateEvent, tableName + ); break; case UPDATE: - handleUpdateEvent(omdbUpdateEvent, tableName, objectCountMap, - unReplicatedSizeMap, replicatedSizeMap); + handleUpdateEvent(omdbUpdateEvent, tableName + ); break; default: @@ -215,11 +221,7 @@ public Pair process(OMUpdateEventBatch events) { } private void handlePutEvent(OMDBUpdateEvent event, - String tableName, - HashMap objectCountMap, - HashMap unReplicatedSizeMap, - HashMap replicatedSizeMap) - throws IOException { + String tableName) { OmTableHandler tableHandler = tableHandlers.get(tableName); if (event.getValue() != null) { if (tableHandler != null) { @@ -234,19 +236,14 @@ private void handlePutEvent(OMDBUpdateEvent event, private void handleDeleteEvent(OMDBUpdateEvent event, - String tableName, - HashMap objectCountMap, - HashMap unReplicatedSizeMap, - HashMap replicatedSizeMap) - throws IOException { + String tableName) { OmTableHandler tableHandler = tableHandlers.get(tableName); if (event.getValue() != null) { if (tableHandler != null) { tableHandler.handleDeleteEvent(event, tableName, objectCountMap, unReplicatedSizeMap, replicatedSizeMap); } else { - String countKey = getTableCountKeyFromTable(tableName); - objectCountMap.computeIfPresent(countKey, + objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count > 0 ? count - 1L : 0L); } } @@ -254,10 +251,7 @@ private void handleDeleteEvent(OMDBUpdateEvent event, private void handleUpdateEvent(OMDBUpdateEvent event, - String tableName, - HashMap objectCountMap, - HashMap unReplicatedSizeMap, - HashMap replicatedSizeMap) { + String tableName) { OmTableHandler tableHandler = tableHandlers.get(tableName); if (event.getValue() != null) { @@ -275,9 +269,6 @@ private void handleUpdateEvent(OMDBUpdateEvent event, * @param dataMap Map containing the updated count and size information. */ private void writeDataToDB(Map dataMap) { - List insertGlobalStats = new ArrayList<>(); - List updateGlobalStats = new ArrayList<>(); - for (Entry entry : dataMap.entrySet()) { Timestamp now = using(sqlConfiguration).fetchValue(select(currentTimestamp())); @@ -295,6 +286,9 @@ private void writeDataToDB(Map dataMap) { globalStatsDao.insert(insertGlobalStats); globalStatsDao.update(updateGlobalStats); + + insertGlobalStats.clear(); + updateGlobalStats.clear(); } /** @@ -303,13 +297,12 @@ private void writeDataToDB(Map dataMap) { * @return The count map containing the counts for each table. */ private HashMap initializeCountMap() { - Collection tables = getTaskTables(); - HashMap objectCountMap = new HashMap<>(tables.size()); + HashMap objCountMap = new HashMap<>(tables.size()); for (String tableName : tables) { String key = getTableCountKeyFromTable(tableName); - objectCountMap.put(key, getValueForKey(key)); + objCountMap.put(key, getValueForKey(key)); } - return objectCountMap; + return objCountMap; } /** @@ -319,10 +312,12 @@ private HashMap initializeCountMap() { * @return The size map containing the size counts for each table. */ private HashMap initializeSizeMap(boolean replicated) { + String tableName; + OmTableHandler tableHandler; HashMap sizeCountMap = new HashMap<>(); for (Map.Entry entry : tableHandlers.entrySet()) { - String tableName = entry.getKey(); - OmTableHandler tableHandler = entry.getValue(); + tableName = entry.getKey(); + tableHandler = entry.getValue(); String key = replicated ? tableHandler.getReplicatedSizeKeyFromTable(tableName) : tableHandler.getUnReplicatedSizeKeyFromTable(tableName); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java index 7a27d29d8f2..5ad51e71147 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OpenKeysInsightHandler.java @@ -48,16 +48,12 @@ public void handlePutEvent(OMDBUpdateEvent event, HashMap unReplicatedSizeMap, HashMap replicatedSizeMap) { - String countKey = getTableCountKeyFromTable(tableName); - String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName); - String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName); - if (event.getValue() != null) { OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue(); - objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L); - unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey, + objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count + 1L); + unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size + omKeyInfo.getDataSize()); - replicatedSizeMap.computeIfPresent(replicatedSizeKey, + replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), (k, size) -> size + omKeyInfo.getReplicatedSize()); } else { LOG.warn("Put event does not have the Key Info for {}.", @@ -76,18 +72,14 @@ public void handleDeleteEvent(OMDBUpdateEvent event, HashMap unReplicatedSizeMap, HashMap replicatedSizeMap) { - String countKey = getTableCountKeyFromTable(tableName); - String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName); - String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName); - if (event.getValue() != null) { OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue(); - objectCountMap.computeIfPresent(countKey, + objectCountMap.computeIfPresent(getTableCountKeyFromTable(tableName), (k, count) -> count > 0 ? count - 1L : 0L); - unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey, + unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size > omKeyInfo.getDataSize() ? size - omKeyInfo.getDataSize() : 0L); - replicatedSizeMap.computeIfPresent(replicatedSizeKey, + replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), (k, size) -> size > omKeyInfo.getReplicatedSize() ? size - omKeyInfo.getReplicatedSize() : 0L); } else { @@ -113,17 +105,15 @@ public void handleUpdateEvent(OMDBUpdateEvent event, event.getKey()); return; } - String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName); - String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName); // In Update event the count for the open table will not change. So we // don't need to update the count. OmKeyInfo oldKeyInfo = (OmKeyInfo) event.getOldValue(); OmKeyInfo newKeyInfo = (OmKeyInfo) event.getValue(); - unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey, + unReplicatedSizeMap.computeIfPresent(getUnReplicatedSizeKeyFromTable(tableName), (k, size) -> size - oldKeyInfo.getDataSize() + newKeyInfo.getDataSize()); - replicatedSizeMap.computeIfPresent(replicatedSizeKey, + replicatedSizeMap.computeIfPresent(getReplicatedSizeKeyFromTable(tableName), (k, size) -> size - oldKeyInfo.getReplicatedSize() + newKeyInfo.getReplicatedSize()); } else {