Merge branch 'master' of https://github.com/apache/samza

Author: sborya

.travis.yml

@@ -30,7 +30,8 @@ jdk:
   - oraclejdk8
 
 script:
-  - ./gradlew clean build
+  ## travis_wait increases build idle-wait time from 10 minutes to 20 minutes.
+  - travis_wait 20 ./gradlew clean build
   - type sonar-scanner &>/dev/null; if [ $? -eq 0 ]; then sonar-scanner; else echo "Not running sonar"; fi
 
 before_cache:

README.md

@@ -1,4 +1,4 @@
-## What is Samza?  [![Build Status](https://builds.apache.org/view/S-Z/view/Samza/job/samza-freestyle-build/badge/icon)](https://builds.apache.org/view/S-Z/view/Samza/job/samza-freestyle-build/) [![Build Status](https://travis-ci.org/apache/samza.svg?branch=master)](https://travis-ci.org/apache/samza)
+## What is Samza?  [![Build Status](https://travis-ci.org/apache/samza.svg?branch=master)](https://travis-ci.org/apache/samza)
 
 [Apache Samza](http://samza.apache.org/) is a distributed stream processing framework. It uses [Apache Kafka](http://kafka.apache.org) for messaging, and [Apache Hadoop YARN](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html) to provide fault tolerance, processor isolation, security, and resource management.
 
@@ -12,7 +12,7 @@ Samza's key features include:
 * **Pluggable:** Though Samza works out of the box with Kafka and YARN, Samza provides a pluggable API that lets you run Samza with other messaging systems and execution environments.
 * **Processor isolation:** Samza works with Apache YARN, which supports Hadoop's security model, and resource isolation through Linux CGroups.
 
-Check out [Hello Samza](https://samza.apache.org/startup/hello-samza/0.11/) to try Samza. Read the [Background](https://samza.apache.org/learn/documentation/0.8/introduction/background.html) page to learn more about Samza.
+Check out [Hello Samza](https://samza.apache.org/startup/hello-samza/latest/) to try Samza. Read the [Background](https://samza.apache.org/learn/documentation/latest/introduction/background.html) page to learn more about Samza.
 
 ### Building Samza
 

bin/check-all.sh

@@ -21,7 +21,7 @@
 
 set -e
 
-SCALAs=( "2.10" "2.11" "2.12" )
+SCALAs=( "2.11" "2.12" )
 JDKs=( "JAVA8_HOME" )
 YARNs=( "2.6.1" "2.7.1" )
 

bin/integration-tests.sh

@@ -53,7 +53,8 @@ cd $ABS_TEST_DIR
 # setup virtualenv locally if it's not already there
 VIRTUAL_ENV=virtualenv-15.0.1
 if [[ ! -d "${ABS_TEST_DIR}/${VIRTUAL_ENV}" ]] ; then
-  curl -O https://pypi.python.org/packages/source/v/virtualenv/$VIRTUAL_ENV.tar.gz
+  # using --location to follow any redirects
+  curl -O --location https://pypi.python.org/packages/source/v/virtualenv/$VIRTUAL_ENV.tar.gz
   tar xvfz $VIRTUAL_ENV.tar.gz
 fi
 

build.gradle

@@ -52,6 +52,7 @@ rat {
     '**/*.svg',
     '**/*.ttf',
     '**/*.woff',
+    '*.log',
     '**/hs_err_pid*.log',
     '**/.classpath',
     '**/.cache/**',
@@ -100,7 +101,7 @@ allprojects {
   // For all scala compilation, add extra compiler options, taken from version-specific
   // dependency-versions-scala file applied above.
   tasks.withType(ScalaCompile) {
-    scalaCompileOptions.additionalParameters = [ scalaOptions ]
+    scalaCompileOptions.additionalParameters = scalaOptions
   }
 }
 
@@ -115,6 +116,11 @@ subprojects {
   apply plugin: 'project-report'
   apply plugin: 'jacoco'
 
+  tasks.withType(ScalaCompile) {
+    // show compile errors in console output
+    logging.setLevel LogLevel.WARN
+  }
+
   tasks.withType(Test) {
     test {
       testLogging {
@@ -197,7 +203,7 @@ project(':samza-azure') {
 
   dependencies {
     compile "com.microsoft.azure:azure-storage:5.3.1"
-    compile "com.microsoft.azure:azure-eventhubs:0.14.5"
+    compile "com.microsoft.azure:azure-eventhubs:1.0.1"
     compile "com.fasterxml.jackson.core:jackson-core:2.8.8"
     compile "io.dropwizard.metrics:metrics-core:3.1.2"
     compile project(':samza-api')
@@ -311,13 +317,17 @@ project(':samza-sql') {
   dependencies {
     compile project(':samza-api')
     compile project(":samza-kafka_$scalaVersion")
+    compile project(":samza-kv-inmemory_$scalaVersion")
+    compile project(":samza-kv-rocksdb_$scalaVersion")
     compile "org.apache.avro:avro:$avroVersion"
     compile "org.apache.calcite:calcite-core:$calciteVersion"
     compile "org.slf4j:slf4j-api:$slf4jVersion"
 
-    testCompile project(":samza-test_$scalaVersion")
     testCompile "junit:junit:$junitVersion"
     testCompile "org.mockito:mockito-core:$mockitoVersion"
+    testCompile "org.powermock:powermock-api-mockito:$powerMockVersion"
+    testCompile "org.powermock:powermock-core:$powerMockVersion"
+    testCompile "org.powermock:powermock-module-junit4:$powerMockVersion"
 
     testRuntime "org.slf4j:slf4j-simple:$slf4jVersion"
   }
@@ -442,6 +452,7 @@ project(":samza-yarn_$scalaVersion") {
     compile "org.scala-lang:scala-compiler:$scalaLibVersion"
     compile "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
     compile "commons-httpclient:commons-httpclient:$commonsHttpClientVersion"
+    compile "org.apache.httpcomponents:httpclient:$httpClientVersion"
     compile("org.apache.hadoop:hadoop-yarn-api:$yarnVersion") {
       exclude module: 'slf4j-log4j12'
     }
@@ -748,21 +759,24 @@ project(":samza-test_$scalaVersion") {
     compile project(":samza-kv-inmemory_$scalaVersion")
     compile project(":samza-kv-rocksdb_$scalaVersion")
     compile project(":samza-core_$scalaVersion")
+    compile project(":samza-kafka_$scalaVersion")
+    compile project(":samza-sql")
     runtime project(":samza-log4j")
     runtime project(":samza-yarn_$scalaVersion")
-    runtime project(":samza-kafka_$scalaVersion")
     runtime project(":samza-hdfs_$scalaVersion")
     compile "org.scala-lang:scala-library:$scalaLibVersion"
     compile "net.sf.jopt-simple:jopt-simple:$joptSimpleVersion"
     compile "javax.mail:mail:1.4"
     compile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion"
+    compile "junit:junit:$junitVersion"
+    compile "org.hamcrest:hamcrest-all:$hamcrestVersion"
     testCompile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion:test"
     testCompile "com.101tec:zkclient:$zkClientVersion"
-    testCompile "junit:junit:$junitVersion"
     testCompile project(":samza-kafka_$scalaVersion")
     testCompile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion:test"
     testCompile "org.apache.kafka:kafka-clients:$kafkaVersion:test"
     testCompile project(":samza-core_$scalaVersion").sourceSets.test.output
+    testCompile project(":samza-sql").sourceSets.test.output
     testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
     testCompile "org.mockito:mockito-core:$mockitoVersion"
     testRuntime "org.slf4j:slf4j-simple:$slf4jVersion"

docs/Gemfile

@@ -14,5 +14,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
 source 'https://rubygems.org'
-gem 'jekyll'
+
+gem 'jekyll', '3.4.5'
+gem 'json', '2.1.0'
+gem 'redcarpet', '3.4.0'
+gem 'pygments.rb', '1.2.1'

docs/Gemfile.lock

@@ -1,62 +1,59 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    blankslate (2.1.2.4)
-    celluloid (0.15.2)
-      timers (~> 1.1.0)
-    classifier (1.3.4)
-      fast-stemmer (>= 1.0.0)
-    coffee-script (2.2.0)
-      coffee-script-source
-      execjs
-    coffee-script-source (1.7.0)
-    colorator (0.1)
-    execjs (2.2.0)
-    fast-stemmer (1.0.2)
-    ffi (1.9.3)
-    jekyll (2.0.3)
-      classifier (~> 1.3)
-      colorator (~> 0.1)
-      jekyll-coffeescript (~> 1.0)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    colorator (1.1.0)
+    ffi (1.9.23)
+    forwardable-extended (2.6.0)
+    jekyll (3.4.5)
+      addressable (~> 2.4)
+      colorator (~> 1.0)
       jekyll-sass-converter (~> 1.0)
+      jekyll-watch (~> 1.1)
       kramdown (~> 1.3)
-      liquid (~> 2.5.5)
-      listen (~> 2.5)
+      liquid (~> 3.0)
       mercenary (~> 0.3.3)
-      pygments.rb (~> 0.5.0)
-      redcarpet (~> 3.1)
+      pathutil (~> 0.9)
+      rouge (~> 1.7)
       safe_yaml (~> 1.0)
-      toml (~> 0.1.0)
-    jekyll-coffeescript (1.0.0)
-      coffee-script (~> 2.2)
-    jekyll-sass-converter (1.0.0)
-      sass (~> 3.2)
-    kramdown (1.4.0)
-    liquid (2.5.5)
-    listen (2.7.8)
-      celluloid (>= 0.15.2)
-      rb-fsevent (>= 0.9.3)
-      rb-inotify (>= 0.9)
-    mercenary (0.3.3)
-    parslet (1.5.0)
-      blankslate (~> 2.0)
-    posix-spawn (0.3.8)
-    pygments.rb (0.5.4)
-      posix-spawn (~> 0.3.6)
-      yajl-ruby (~> 1.1.0)
-    rb-fsevent (0.9.4)
-    rb-inotify (0.9.5)
-      ffi (>= 0.5.0)
-    redcarpet (3.1.2)
+    jekyll-sass-converter (1.5.2)
+      sass (~> 3.4)
+    jekyll-watch (1.5.0)
+      listen (~> 3.0, < 3.1)
+    json (2.1.0)
+    kramdown (1.16.2)
+    liquid (3.0.6)
+    listen (3.0.8)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    mercenary (0.3.6)
+    multi_json (1.13.1)
+    pathutil (0.16.1)
+      forwardable-extended (~> 2.6)
+    public_suffix (2.0.5)
+    pygments.rb (1.2.1)
+      multi_json (>= 1.0.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    redcarpet (3.4.0)
+    rouge (1.11.1)
     safe_yaml (1.0.4)
-    sass (3.3.8)
-    timers (1.1.0)
-    toml (0.1.1)
-      parslet (~> 1.5.0)
-    yajl-ruby (1.1.0)
+    sass (3.5.6)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  jekyll
+  jekyll (= 3.4.5)
+  json (= 2.1.0)
+  pygments.rb (= 1.2.1)
+  redcarpet (= 3.4.0)
+
+BUNDLED WITH
+   1.16.1

docs/README.md

@@ -24,15 +24,15 @@ Samza's documentation uses Jekyll to build a website out of markdown pages. Prer
 
 To serve the website on [localhost:4000](http://localhost:4000/):
 
-    bundle exec jekyll serve --watch --baseurl
+    bundle exec jekyll serve --watch --baseurl ""
 
 To compile the website in the \_site directory, execute:
 
     bundle exec jekyll build
 
 To test the site, run:
 
-    bundle exec jekyll serve --watch --baseurl
+    bundle exec jekyll serve --watch --baseurl ""
 
 ## Versioning
 

docs/community/committers.md

@@ -30,17 +30,17 @@ Committer, and PMC member<br/>
 <a href="https://twitter.com/navina_r" target="_blank"><i class="fa fa-twitter committer-icon"></i></a>
 
 **Jagadish Venkatraman**<br/>
-Committer<br/>
+Committer, and PMC member<br/>
 <a href="https://www.linkedin.com/in/jagadishvenkat" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>
 <a href="https://twitter.com/vjagadish1989" target="_blank"><i class="fa fa-twitter committer-icon"></i></a>
 
 **Jake Maes**<br/>
-Committer<br/>
+Committer, and PMC member<br/>
 <a href="www.linkedin.com/in/jacobmaes" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>
 <a href="https://twitter.com/jakemaes" target="_blank"><i class="fa fa-twitter committer-icon"></i></a>
 
 **Xinyu Liu**<br/>
-Committer<br/>
+Committer, and PMC member<br/>
 <a href="https://www.linkedin.com/in/xinyu-liu-b0b21648" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>
 
 **Chris Riccomini**<br/>
@@ -95,4 +95,4 @@ Committer<br/>
 
 **Prateek Maheshwari**<br/>
 Committer<br/>
-<a href="https://www.linkedin.com/in/mprateek" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>
+<a href="https://www.linkedin.com/in/mprateek" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>

docs/learn/documentation/versioned/azure/eventhubs.md

@@ -58,6 +58,10 @@ collector.send(envelope);
 
 Each [OutgoingMessageEnvelope](https://samza.apache.org/learn/documentation/latest/api/javadocs/org/apache/samza/system/OutgoingMessageEnvelope.html) is converted into an [EventData](https://docs.microsoft.com/en-us/java/api/com.microsoft.azure.eventhubs._event_data) instance whose body is set to the `message` in the envelope. Additionally, the `key` and the `produce timestamp` are set as properties in the EventData before sending it to EventHubs.
 
+#### Size limit of partition key:
+
+Note that EventHubs has a limit on the length of partition key (128 characters). In [EventHubSystemProducer](https://github.com/apache/samza/blob/master/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/EventHubSystemProducer.java) we truncate the partition key if the size of the key exceeds the limit.
+
 ### Advanced configuration:
 
 ##### Producer partitioning: 
@@ -165,7 +169,7 @@ With the environment setup complete, let us move on to building the hello-samza
 ```
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.14.1-SNAPSHOT-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.15.0-SNAPSHOT-dist.tar.gz -C deploy/samza
 ```
 
 We are now all set to deploy the application locally.