Merge pull request #10 from sharding-sphere/dev

update from origin

README.md

@@ -11,10 +11,10 @@
 
 ## Document
 
-[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](http://shardingsphere.io/document/en/)
+[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](http://shardingsphere.io/document/current/en/)
 [![Roadmap](https://img.shields.io/badge/roadmap-English-blue.svg)](ROADMAP.md)
 
-[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](http://shardingsphere.io/document/cn/)
+[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](http://shardingsphere.io/document/current/cn/)
 
 ## Overview
 

README_ZH.md

@@ -11,7 +11,7 @@
 
 ## 文档
 
-[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](http://shardingsphere.io/document/cn/)
+[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](http://shardingsphere.io/document/current/cn/)
 [![Roadmap](https://img.shields.io/badge/roadmap-English-blue.svg)](ROADMAP.md)
 
 ## 概述

sharding-core/src/main/java/io/shardingsphere/core/constant/ShardingPropertiesConstant.java

@@ -51,7 +51,21 @@ public enum ShardingPropertiesConstant {
      * Default: same with CPU cores.
      * </p>
      */
-    EXECUTOR_SIZE("executor.size", String.valueOf(Runtime.getRuntime().availableProcessors()), int.class);
+    EXECUTOR_SIZE("executor.size", String.valueOf(Runtime.getRuntime().availableProcessors()), int.class),
+
+    /**
+     * Proxy mode.
+     *
+     * <p>
+     * MEMORY_STRICTLY:
+     * Proxy holds as many connections as the count of actual tables routed in a database.
+     * The benefit of this approach is saving memory for Proxy by Stream ResultSet.
+     * CONNECTION_STRICTLY:
+     * Proxy will release connections after get the overall rows from the ResultSet.
+     * Meanwhile, the cost of the memory will be increased.
+     * </p>
+     */
+    PROXY_MODE("proxy.mode", "MEMORY_STRICTLY", String.class);
 
     private final String key;
 

sharding-core/src/main/java/io/shardingsphere/core/metadata/ShardingMetaData.java

@@ -17,21 +17,32 @@
 
 package io.shardingsphere.core.metadata;
 
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import io.shardingsphere.core.exception.ShardingException;
 import io.shardingsphere.core.rule.DataNode;
 import io.shardingsphere.core.rule.ShardingDataSourceNames;
 import io.shardingsphere.core.rule.ShardingRule;
 import io.shardingsphere.core.rule.TableRule;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Abstract Sharding metadata.
@@ -41,9 +52,20 @@
  */
 @Getter
 @Setter
-public abstract class ShardingMetaData {
+@Slf4j
+public abstract class ShardingMetaData implements AutoCloseable {
 
+    private static final ThreadPoolExecutor THREAD_POOL_EXECUTOR = new ThreadPoolExecutor(
+            Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Sharding-JDBC-%d").build());
+
+    private final ListeningExecutorService executorService;
+
     private Map<String, TableMetaData> tableMetaDataMap;
+
+    public ShardingMetaData() {
+        executorService = MoreExecutors.listeningDecorator(THREAD_POOL_EXECUTOR);
+        MoreExecutors.addDelayedShutdownHook(executorService, 60, TimeUnit.SECONDS);
+    }
 
     /**
      * Initialize sharding metadata.
@@ -78,25 +100,40 @@ public void refresh(final TableRule each, final ShardingRule shardingRule) throw
      * @throws SQLException SQL exception
      */
     public void refresh(final TableRule each, final ShardingRule shardingRule, final Map<String, Connection> connectionMap) throws SQLException {
-        tableMetaDataMap.put(each.getLogicTable(), getTableMetaData(each.getLogicTable(), each.getActualDataNodes(), shardingRule.getShardingDataSourceNames(), connectionMap));
+        tableMetaDataMap.put(each.getLogicTable(), getFinalTableMetaData(each.getLogicTable(), each.getActualDataNodes(), shardingRule.getShardingDataSourceNames(), connectionMap));
     }
 
-    private TableMetaData getTableMetaData(final String logicTableName, final List<DataNode> actualDataNodes,
-                                           final ShardingDataSourceNames shardingDataSourceNames, final Map<String, Connection> connectionMap) throws SQLException {
-        Collection<ColumnMetaData> result = null;
-        for (DataNode each : actualDataNodes) {
-            Collection<ColumnMetaData> columnMetaDataList = getColumnMetaDataList(each, shardingDataSourceNames, connectionMap);
-            if (null == result) {
-                result = columnMetaDataList;
+    private TableMetaData getFinalTableMetaData(final String logicTableName, final List<DataNode> actualDataNodes,
+                                                final ShardingDataSourceNames shardingDataSourceNames, final Map<String, Connection> connectionMap) throws SQLException {
+        List<TableMetaData> actualTableMetaDataList = getAllActualTableMetaData(actualDataNodes, shardingDataSourceNames, connectionMap);
+        for (int i = 0; i < actualTableMetaDataList.size(); i++) {
+            if (actualTableMetaDataList.size() - 1 == i) {
+                return actualTableMetaDataList.get(i);
             }
-            if (!result.equals(columnMetaDataList)) {
-                throw new ShardingException(getErrorMsgOfTableMetaData(logicTableName, result, columnMetaDataList));
+            if (!actualTableMetaDataList.get(i).equals(actualTableMetaDataList.get(i + 1))) {
+                throw new ShardingException(getErrorMsgOfTableMetaData(logicTableName, actualTableMetaDataList.get(i), actualTableMetaDataList.get(i + 1)));
             }
         }
-        return new TableMetaData(result);
+        return new TableMetaData(new ArrayList<ColumnMetaData>());
     }
-
-    /**
+
+    private List<TableMetaData> getAllActualTableMetaData(final List<DataNode> actualDataNodes, final ShardingDataSourceNames shardingDataSourceNames, final Map<String, Connection> connectionMap) throws SQLException {
+        List<ListenableFuture<TableMetaData>> result = new ArrayList<>();
+        for (final DataNode each : actualDataNodes) {
+            result.add(executorService.submit(new Callable<TableMetaData>() {
+                public TableMetaData call() throws Exception {
+                    return getTableMetaData(each, shardingDataSourceNames, connectionMap);
+                }
+            }));
+        }
+        try {
+            return Futures.allAsList(result).get();
+        } catch (final InterruptedException | ExecutionException ex) {
+            throw new ShardingException(ex);
+        }
+    }
+
+    /**d
      * Get column metadata implementing by concrete handler.
      *
      * @param dataNode DataNode
@@ -105,19 +142,33 @@ private TableMetaData getTableMetaData(final String logicTableName, final List<D
      * @return ColumnMetaData
      * @throws SQLException SQL exception
      */
-    public abstract Collection<ColumnMetaData> getColumnMetaDataList(DataNode dataNode, ShardingDataSourceNames shardingDataSourceNames, Map<String, Connection> connectionMap) throws SQLException;
+    public abstract TableMetaData getTableMetaData(DataNode dataNode, ShardingDataSourceNames shardingDataSourceNames, Map<String, Connection> connectionMap) throws SQLException;
 
-    private String getErrorMsgOfTableMetaData(final String logicTableName, final Collection<ColumnMetaData> oldColumnMetaDataList, final Collection<ColumnMetaData> newColumnMetaDataList) {
+    private String getErrorMsgOfTableMetaData(final String logicTableName, final TableMetaData oldTableMetaData, final TableMetaData newTableMetaData) {
         StringBuilder stringBuilder = new StringBuilder();
         stringBuilder.append(" Cannot get uniformed table structure for ").append(logicTableName).append(".");
-        stringBuilder.append(" The different column metadata of actual tables is as follows: ");
-        for (ColumnMetaData each : oldColumnMetaDataList) {
-            stringBuilder.append(each.toString()).append(" ");
-        }
+        stringBuilder.append("The different metadata of actual tables is as follows:");
+        stringBuilder.append(oldTableMetaData.toString());
         stringBuilder.append("\n");
-        for (ColumnMetaData each : newColumnMetaDataList) {
-            stringBuilder.append(each.toString()).append(" ");
-        }
+        stringBuilder.append(newTableMetaData.toString());
         return stringBuilder.toString();
     }
+
+    @Override
+    public void close() {
+        THREAD_POOL_EXECUTOR.execute(new Runnable() {
+
+            @Override
+            public void run() {
+                try {
+                    executorService.shutdown();
+                    while (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
+                        executorService.shutdownNow();
+                    }
+                } catch (final InterruptedException ex) {
+                    log.error("ExecutorEngine can not been terminated", ex);
+                }
+            }
+        });
+    }
 }

sharding-core/src/main/java/io/shardingsphere/core/metadata/TableMetaData.java

@@ -20,6 +20,7 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.ToString;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -33,6 +34,7 @@
 @RequiredArgsConstructor
 @Getter
 @EqualsAndHashCode
+@ToString
 public final class TableMetaData {
 
     private final Collection<ColumnMetaData> columnMetaData;

sharding-core/src/main/java/io/shardingsphere/core/parsing/parser/sql/dql/select/AbstractSelectParser.java

@@ -20,6 +20,7 @@
 import com.google.common.base.Optional;
 import io.shardingsphere.core.constant.AggregationType;
 import io.shardingsphere.core.metadata.ShardingMetaData;
+import io.shardingsphere.core.metadata.TableMetaData;
 import io.shardingsphere.core.parsing.lexer.LexerEngine;
 import io.shardingsphere.core.parsing.lexer.token.Assist;
 import io.shardingsphere.core.parsing.lexer.token.DefaultKeyword;
@@ -204,8 +205,9 @@ private boolean isContainsItemInStarSelectItem(final List<StarSelectItem> starSe
             if (orderItem.getOwner().isPresent() && tables.find(orderItem.getOwner().get()).equals(tableOptionalOfStarSelectItem)) {
                 return true;
             }
-            List<String> columnNames = tableOptionalOfStarSelectItem.isPresent()
-                    ? shardingMetaData.getTableMetaDataMap().get(tableOptionalOfStarSelectItem.get().getName()).getAllColumnNames() : new ArrayList<String>();
+            Optional<TableMetaData> tableMetaDataOptional = tableOptionalOfStarSelectItem.isPresent()
+                    ? Optional.fromNullable(shardingMetaData.getTableMetaDataMap().get(tableOptionalOfStarSelectItem.get().getName())) : Optional.<TableMetaData>absent();
+            List<String> columnNames = tableMetaDataOptional.isPresent() ? tableMetaDataOptional.get().getAllColumnNames() : new ArrayList<String>();
             if (columnNames.contains(orderItem.getName().get().toUpperCase()) || columnNames.contains(orderItem.getName().get().toLowerCase())) {
                 return true;
             }

sharding-core/src/main/java/io/shardingsphere/core/routing/strategy/standard/StandardShardingStrategy.java

@@ -76,7 +76,10 @@ private Collection<String> doSharding(final Collection<String> availableTargetNa
     private Collection<String> doSharding(final Collection<String> availableTargetNames, final ListShardingValue<?> shardingValue) {
         Collection<String> result = new LinkedList<>();
         for (PreciseShardingValue<?> each : transferToPreciseShardingValues(shardingValue)) {
-            result.add(preciseShardingAlgorithm.doSharding(availableTargetNames, each));
+            String target = preciseShardingAlgorithm.doSharding(availableTargetNames, each);
+            if (null != target) {
+                result.add(target);
+            }
         }
         return result;
     }

sharding-core/src/main/java/io/shardingsphere/core/yaml/sharding/DataSourceParameter.java

@@ -36,4 +36,14 @@ public final class DataSourceParameter {
     private String username;
 
     private String password;
+
+    private Boolean autoCommit;
+
+    private long connectionTimeout;
+
+    private long idleTimeout;
+
+    private long maxLifetime;
+
+    private int maximumPoolSize;
 }

sharding-core/src/test/java/io/shardingsphere/core/parsing/integrate/asserts/ParserResultSetLoader.java

@@ -86,4 +86,13 @@ public ParserResult getParserResult(final String sqlCaseId) {
         Preconditions.checkState(parserResultMap.containsKey(sqlCaseId), "Can't find SQL of id: " + sqlCaseId);
         return parserResultMap.get(sqlCaseId);
     }
+
+    /**
+     * Count all parser test cases.
+     *
+     * @return count of all parser test cases
+     */
+    public int countAllParserTestCases() {
+        return parserResultMap.size();
+    }
 }

sharding-core/src/test/java/io/shardingsphere/core/parsing/integrate/engine/IntegrateSupportedSQLParsingTest.java

@@ -30,6 +30,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
 @RequiredArgsConstructor
 public final class IntegrateSupportedSQLParsingTest extends AbstractBaseIntegrateSQLParsingTest {
 
@@ -45,6 +48,7 @@ public final class IntegrateSupportedSQLParsingTest extends AbstractBaseIntegrat
 
     @Parameters(name = "{0} ({2}) -> {1}")
     public static Collection<Object[]> getTestParameters() {
+        assertThat(sqlCasesLoader.countAllSupportedSQLCases(), is(parserResultSetLoader.countAllParserTestCases()));
         return sqlCasesLoader.getSupportedSQLTestParameters(Arrays.<Enum>asList(DatabaseType.values()), DatabaseType.class);
     }
 

sharding-jdbc/src/main/java/io/shardingsphere/core/jdbc/metadata/JDBCShardingMetaData.java

@@ -18,8 +18,8 @@
 package io.shardingsphere.core.jdbc.metadata;
 
 import io.shardingsphere.core.constant.DatabaseType;
-import io.shardingsphere.core.metadata.ColumnMetaData;
 import io.shardingsphere.core.metadata.ShardingMetaData;
+import io.shardingsphere.core.metadata.TableMetaData;
 import io.shardingsphere.core.rule.DataNode;
 import io.shardingsphere.core.rule.ShardingDataSourceNames;
 import io.shardingsphere.core.rule.ShardingRule;
@@ -29,7 +29,6 @@
 import javax.sql.DataSource;
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.Collection;
 import java.util.Map;
 
 /**
@@ -48,13 +47,13 @@ public final class JDBCShardingMetaData extends ShardingMetaData {
     private final DatabaseType databaseType;
 
     @Override
-    public Collection<ColumnMetaData> getColumnMetaDataList(final DataNode dataNode, final ShardingDataSourceNames shardingDataSourceNames,
-                                                            final Map<String, Connection> connectionMap) throws SQLException {
+    public TableMetaData getTableMetaData(final DataNode dataNode, final ShardingDataSourceNames shardingDataSourceNames,
+                                          final Map<String, Connection> connectionMap) throws SQLException {
         String dataSourceName = shardingDataSourceNames.getRawMasterDataSourceName(dataNode.getDataSourceName());
         if (connectionMap.containsKey(dataSourceName)) {
-            return ShardingMetaDataHandlerFactory.newInstance(dataNode.getTableName(), databaseType).getColumnMetaDataList(connectionMap.get(dataSourceName));
+            return ShardingMetaDataHandlerFactory.newInstance(dataNode.getTableName(), databaseType).getTableMetaData(connectionMap.get(dataSourceName));
         } else {
-            return ShardingMetaDataHandlerFactory.newInstance(dataSourceMap.get(dataSourceName), dataNode.getTableName(), databaseType).getColumnMetaDataList();
+            return ShardingMetaDataHandlerFactory.newInstance(dataSourceMap.get(dataSourceName), dataNode.getTableName(), databaseType).getTableMetaData();
         }
     }
 }

sharding-jdbc/src/main/java/io/shardingsphere/core/jdbc/metadata/dialect/ShardingMetaDataHandler.java

@@ -18,15 +18,15 @@
 package io.shardingsphere.core.jdbc.metadata.dialect;
 
 import io.shardingsphere.core.metadata.ColumnMetaData;
+import io.shardingsphere.core.metadata.TableMetaData;
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
 import javax.sql.DataSource;
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.Collection;
-import java.util.LinkedList;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -44,34 +44,32 @@ public abstract class ShardingMetaDataHandler {
     private final String actualTableName;
 
     /**
-     * Get column meta data list.
+     * Get table meta data.
      *
-     * @return column meta data list
+     * @return table meta data
      * @throws SQLException SQL exception
      */
-    public Collection<ColumnMetaData> getColumnMetaDataList() throws SQLException {
-        List<ColumnMetaData> result = new LinkedList<>();
+    public TableMetaData getTableMetaData() throws SQLException {
         try (Connection connection = dataSource.getConnection()) {
             if (isTableExist(connection)) {
-                result = getExistColumnMeta(connection);
+                return new TableMetaData(getExistColumnMeta(connection));
             }
         }
-        return result;
+        return new TableMetaData(new ArrayList<ColumnMetaData>());
     }
 
     /**
-     * Get column metadata by Sharding Connection.
+     * Get table metadata by Sharding Connection.
      *
      * @param connection connection
-     * @return column metadata List
+     * @return table metadata
      * @throws SQLException SQL exception
      */
-    public Collection<ColumnMetaData> getColumnMetaDataList(final Connection connection) throws SQLException {
-        List<ColumnMetaData> result = new LinkedList<>();
+    public TableMetaData getTableMetaData(final Connection connection) throws SQLException {
         if (isTableExist(connection)) {
-            result = getExistColumnMeta(connection);
+            return new TableMetaData(getExistColumnMeta(connection));
         }
-        return result;
+        return new TableMetaData(new ArrayList<ColumnMetaData>());
     }
 
     /**