init

sentinel/sentinel-extension/sentinel-datasource-consul/README.md

@@ -0,0 +1,28 @@
+# Sentinel DataSource Consul
+
+Sentinel DataSource Consul provides integration with Consul. The data source leverages blocking query (backed by
+long polling) of Consul.
+
+## Usage
+
+To use Sentinel DataSource Consul, you could add the following dependency:
+
+```xml
+<dependency>
+    <groupId>com.alibaba.csp</groupId>
+    <artifactId>sentinel-datasource-consul</artifactId>
+    <version>x.y.z</version>
+</dependency>
+
+```
+
+Then you can create a `ConsulDataSource` and register to rule managers.
+For instance:
+
+```java
+ReadableDataSource<String, List<FlowRule>> dataSource = new ConsulDataSource<>(host, port, ruleKey, waitTimeoutInSecond, flowConfigParser);
+FlowRuleManager.register2Property(dataSource.getProperty());
+```
+
+- `ruleKey`: the rule persistence key
+- `waitTimeoutInSecond`: long polling timeout (in second) of the Consul API client

sentinel/sentinel-extension/sentinel-datasource-consul/pom.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>sentinel-extension</artifactId>
+        <groupId>com.alibaba.csp</groupId>
+        <version>1.8.3</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>sentinel-datasource-consul</artifactId>
+    <packaging>jar</packaging>
+
+    <properties>
+        <java.source.version>1.8</java.source.version>
+        <java.target.version>1.8</java.target.version>
+        <consul.version>1.4.5</consul.version>
+        <consul.process.version>2.2.0</consul.process.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.codehaus.groovy</groupId>
+            <artifactId>groovy-xml</artifactId>
+            <version>3.0.6</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.csp</groupId>
+            <artifactId>sentinel-datasource-extension</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.ecwid.consul</groupId>
+            <artifactId>consul-api</artifactId>
+            <version>${consul.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.pszymczyk.consul</groupId>
+            <artifactId>embedded-consul</artifactId>
+            <version>${consul.process.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>

sentinel/sentinel-extension/sentinel-datasource-consul/src/main/java/com/alibaba/csp/sentinel/datasource/consul/ConsulDataSource.java

@@ -0,0 +1,225 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.datasource.consul;
+
+import com.alibaba.csp.sentinel.concurrent.NamedThreadFactory;
+import com.alibaba.csp.sentinel.datasource.AbstractDataSource;
+import com.alibaba.csp.sentinel.datasource.Converter;
+import com.alibaba.csp.sentinel.log.RecordLog;
+import com.alibaba.csp.sentinel.util.AssertUtil;
+
+import com.alibaba.csp.sentinel.util.StringUtil;
+import com.ecwid.consul.v1.ConsulClient;
+import com.ecwid.consul.v1.QueryParams;
+import com.ecwid.consul.v1.Response;
+import com.ecwid.consul.v1.kv.model.GetValue;
+
+import java.util.concurrent.*;
+
+/**
+ * <p>
+ * A read-only {@code DataSource} with Consul backend.
+ * <p>
+ * <p>
+ * The data source first initial rules from a Consul during initialization.
+ * Then it start a watcher to observe the updates of rule date and update to memory.
+ *
+ * Consul do not provide http api to watch the update of KV，so it use a long polling and
+ * <a href="https://www.consul.io/api/features/blocking.html">blocking queries</a> of the Consul's feature
+ * to watch and update value easily.When Querying data by index will blocking until change or timeout. If
+ * the index of the current query is larger than before, it means that the data has changed.
+ * </p>
+ *
+ * @author wavesZh
+ * @author Zhiguo.Chen
+ */
+public class ConsulDataSource<T> extends AbstractDataSource<String, T> {
+
+    private static final int DEFAULT_PORT = 8500;
+
+    private final String address;
+    private final String token;
+    private final String ruleKey;
+    /**
+     * Request of query will hang until timeout (in second) or get updated value.
+     */
+    private final int watchTimeout;
+
+    /**
+     * Record the data's index in Consul to watch the change.
+     * If lastIndex is smaller than the index of next query, it means that rule data has updated.
+     */
+    private volatile long lastIndex;
+
+    private final ConsulClient client;
+
+    private final ConsulKVWatcher watcher = new ConsulKVWatcher();
+
+    @SuppressWarnings("PMD.ThreadPoolCreationRule")
+    private final ExecutorService watcherService = Executors.newSingleThreadExecutor(
+        new NamedThreadFactory("sentinel-consul-ds-watcher", true));
+
+    public ConsulDataSource(String host, String ruleKey, int watchTimeoutInSecond, Converter<String, T> parser) {
+        this(host, DEFAULT_PORT, ruleKey, watchTimeoutInSecond, parser);
+    }
+
+    /**
+     * Constructor of {@code ConsulDataSource}.
+     *
+     * @param parser       customized data parser, cannot be empty
+     * @param host         consul agent host
+     * @param port         consul agent port
+     * @param ruleKey      data key in Consul
+     * @param watchTimeout request for querying data will be blocked until new data or timeout. The unit is second (s)
+     */
+    public ConsulDataSource(String host, int port, String ruleKey, int watchTimeout, Converter<String, T> parser) {
+        this(host, port, null, ruleKey, watchTimeout, parser);
+    }
+
+    /**
+     * Constructor of {@code ConsulDataSource}.
+     *
+     * @param parser       customized data parser, cannot be empty
+     * @param host         consul agent host
+     * @param port         consul agent port
+     * @param token     consul agent acl token
+     * @param ruleKey      data key in Consul
+     * @param watchTimeout request for querying data will be blocked until new data or timeout. The unit is second (s)
+     */
+    public ConsulDataSource(String host, int port, String token, String ruleKey, int watchTimeout, Converter<String, T> parser) {
+        super(parser);
+        AssertUtil.notNull(host, "Consul host can not be null");
+        AssertUtil.notEmpty(ruleKey, "Consul ruleKey can not be empty");
+        AssertUtil.isTrue(watchTimeout >= 0, "watchTimeout should not be negative");
+        this.client = new ConsulClient(host, port);
+        this.address = host + ":" + port;
+        this.token = token;
+        this.ruleKey = ruleKey;
+        this.watchTimeout = watchTimeout;
+        loadInitialConfig();
+        startKVWatcher();
+    }
+
+    private void startKVWatcher() {
+        watcherService.submit(watcher);
+    }
+
+    private void loadInitialConfig() {
+        try {
+            T newValue = loadConfig();
+            if (newValue == null) {
+                RecordLog.warn(
+                    "[ConsulDataSource] WARN: initial config is null, you may have to check your data source");
+            }
+            getProperty().updateValue(newValue);
+        } catch (Exception ex) {
+            RecordLog.warn("[ConsulDataSource] Error when loading initial config", ex);
+        }
+    }
+
+    @Override
+    public String readSource() throws Exception {
+        if (this.client == null) {
+            throw new IllegalStateException("Consul has not been initialized or error occurred");
+        }
+        Response<GetValue> response = getValueImmediately(ruleKey);
+        if (response != null) {
+            GetValue value = response.getValue();
+            lastIndex = response.getConsulIndex();
+            return value != null ? value.getDecodedValue() : null;
+        }
+        return null;
+    }
+
+    @Override
+    public void close() throws Exception {
+        watcher.stop();
+        watcherService.shutdown();
+    }
+
+    private class ConsulKVWatcher implements Runnable {
+        private volatile boolean running = true;
+
+        @Override
+        public void run() {
+            while (running) {
+                // It will be blocked until watchTimeout(s) if rule data has no update.
+                Response<GetValue> response = getValue(ruleKey, lastIndex, watchTimeout);
+                if (response == null) {
+                    try {
+                        TimeUnit.MILLISECONDS.sleep(watchTimeout * 1000);
+                    } catch (InterruptedException e) {
+                    }
+                    continue;
+                }
+                GetValue getValue = response.getValue();
+                Long currentIndex = response.getConsulIndex();
+                if (currentIndex == null || currentIndex <= lastIndex) {
+                    continue;
+                }
+                lastIndex = currentIndex;
+                if (getValue != null) {
+                    String newValue = getValue.getDecodedValue();
+                    try {
+                        getProperty().updateValue(parser.convert(newValue));
+                        RecordLog.info("[ConsulDataSource] New property value received for ({}, {}): {}",
+                            address, ruleKey, newValue);
+                    } catch (Exception ex) {
+                        // In case of parsing error.
+                        RecordLog.warn("[ConsulDataSource] Failed to update value for ({}, {}), raw value: {}",
+                            address, ruleKey, newValue);
+                    }
+                }
+            }
+        }
+
+        private void stop() {
+            running = false;
+        }
+    }
+
+    /**
+     * Get data from Consul immediately.
+     *
+     * @param key data key in Consul
+     * @return the value associated to the key, or null if error occurs
+     */
+    private Response<GetValue> getValueImmediately(String key) {
+        return getValue(key, -1, -1);
+    }
+
+    /**
+     * Get data from Consul (blocking).
+     *
+     * @param key      data key in Consul
+     * @param index    the index of data in Consul.
+     * @param waitTime time(second) for waiting get updated value.
+     * @return the value associated to the key, or null if error occurs
+     */
+    private Response<GetValue> getValue(String key, long index, long waitTime) {
+        try {
+            if (StringUtil.isNotBlank(token)) {
+                return client.getKVValue(key, token, new QueryParams(waitTime, index));
+            } else {
+                return client.getKVValue(key, new QueryParams(waitTime, index));
+            }
+        } catch (Throwable t) {
+            RecordLog.warn("[ConsulDataSource] Failed to get value for key: " + key, t);
+        }
+        return null;
+    }
+
+}

sentinel/sentinel-extension/sentinel-datasource-consul/src/test/java/com/alibaba/csp/sentinel/datasource/consul/ConsulDataSourceTest.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.datasource.consul;
+
+import com.alibaba.csp.sentinel.datasource.Converter;
+import com.alibaba.csp.sentinel.datasource.ReadableDataSource;
+import com.alibaba.csp.sentinel.slots.block.flow.FlowRule;
+import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+
+import com.ecwid.consul.v1.ConsulClient;
+import com.ecwid.consul.v1.Response;
+import com.pszymczyk.consul.ConsulProcess;
+import com.pszymczyk.consul.ConsulStarterBuilder;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author wavesZh
+ */
+public class ConsulDataSourceTest {
+
+    private final String ruleKey = "sentinel.rules.flow.ruleKey";
+    private final int waitTimeoutInSecond = 1;
+
+    private ConsulProcess consul;
+    private ConsulClient client;
+
+    private ReadableDataSource<String, List<FlowRule>> consulDataSource;
+
+    private List<FlowRule> rules;
+
+    @Before
+    public void init() {
+        this.consul = ConsulStarterBuilder.consulStarter()
+            .build()
+            .start();
+        int port = consul.getHttpPort();
+        String host = "127.0.0.1";
+        client = new ConsulClient(host, port);
+        Converter<String, List<FlowRule>> flowConfigParser = buildFlowConfigParser();
+        String flowRulesJson =
+            "[{\"resource\":\"test\", \"limitApp\":\"default\", \"grade\":1, \"count\":\"0.0\", \"strategy\":0, "
+                + "\"refResource\":null, "
+                +
+                "\"controlBehavior\":0, \"warmUpPeriodSec\":10, \"maxQueueingTimeMs\":500, \"controller\":null}]";
+        initConsulRuleData(flowRulesJson);
+        rules = flowConfigParser.convert(flowRulesJson);
+        consulDataSource = new ConsulDataSource<>(host, port, ruleKey, waitTimeoutInSecond, flowConfigParser);
+        FlowRuleManager.register2Property(consulDataSource.getProperty());
+    }
+
+    @After
+    public void clean() throws Exception {
+        if (consulDataSource != null) {
+            consulDataSource.close();
+        }
+        if (consul != null) {
+            consul.close();
+        }
+        FlowRuleManager.loadRules(new ArrayList<>());
+    }
+
+    @Test
+    public void testConsulDataSourceWhenInit() {
+        List<FlowRule> rules = FlowRuleManager.getRules();
+        Assert.assertEquals(this.rules, rules);
+    }
+
+    @Test
+    public void testConsulDataSourceWhenUpdate() throws InterruptedException {
+        rules.get(0).setMaxQueueingTimeMs(new Random().nextInt());
+        client.setKVValue(ruleKey, JSON.toJSONString(rules));
+        TimeUnit.SECONDS.sleep(waitTimeoutInSecond);
+        List<FlowRule> rules = FlowRuleManager.getRules();
+        Assert.assertEquals(this.rules, rules);
+    }
+
+    private Converter<String, List<FlowRule>> buildFlowConfigParser() {
+        return source -> JSON.parseObject(source, new TypeReference<List<FlowRule>>() {});
+    }
+
+    private void initConsulRuleData(String flowRulesJson) {
+        Response<Boolean> response = client.setKVValue(ruleKey, flowRulesJson);
+        Assert.assertEquals(Boolean.TRUE, response.getValue());
+    }
+
+}