Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Beeline hive job plugin added #289

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Beeline hive job plugin added #289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f8a2f23
BeelineHive plugin added
aycaacar Apr 25, 2018
2d0b6a3
beelineHiveJob test files added
aycaacar Apr 25, 2018
1e5e163
beelinehive jobtype properties file added
aycaacar Apr 25, 2018
c57a752
sets jobtype.classpath in private.properties
aycaacar May 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added extlib/hive-beeline-0.12.0.jar
View file
Open in desktop
Binary file not shown.
361 changes: 184 additions & 177 deletions plugins/jobtype/build.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,201 +1,208 @@
<?xml version="1.0"?>

<project name="azkaban-jobtype" basedir="." default="all">
<property file="build.properties" />
<property name="base.dir" value="${basedir}/../.." />
<property name="dist.dir" value="${base.dir}/dist/jobtype" />
<property name="lib.dir" value="${basedir}/lib" />
<property name="ext.lib.dir" value="${base.dir}/extlib" />
<property name="test.lib.dir" value="${base.dir}/test_lib" />
<property name="test.resource.dir" value="${base.dir}/test_resource" />
<property name="dist.jar.dir" value="${dist.dir}/jars" />
<property name="dist.classes.dir" value="${dist.dir}/classes" />
<property name="dist.classes.test.dir" value="${dist.dir}/testclasses" />
<property name="dist.packages.dir" value="${dist.dir}/packages" />
<property name="test.report.dir" value="${dist.dir}/report" />
<property name="java.src.dir" value="${basedir}/src" />
<property name="java.src.test.dir" value="${basedir}/test" />
<property name="azkaban-jobtype-jar" value="${dist.jar.dir}/${name}-${version}.jar" />
<property name="jobtypes.dir" value="${basedir}/jobtypes" />
<property name="javatype.dir" value="${basedir}/jobtypes/java" />
<property name="pigtype.dir" value="${basedir}/jobtypes/pig" />
<property file="build.properties" />
<property name="hadoopsecuritymanagerjar" value="${base.dir}/dist/hadoopsecuritymanager/jars/azkaban-hadoopsecuritymanager-${version}.jar"/>
<property name="hadoopsecuritymanagerjar-yarn" value="${base.dir}/dist/hadoopsecuritymanager-yarn/jars/azkaban-hadoopsecuritymanageryarn-${version}.jar"/>
<property name="hadoopsecuritymanager-common.java.src.dir" value="${base.dir}/plugins/hadoopsecuritymanager-common/src" />
<property name="crypto.jar.dir" value="${base.dir}/dist/crypto" />
<property name="crypto.jar" value="${base.dir}/dist/crypto/jars/azkaban-crypto-${version}.jar" />

<property name="version.file" value="${base.dir}/dist/package.version" />
<property environment="env" />

<path id="main.classpath">
<fileset dir="${lib.dir}">
<include name="*.jar" />
</fileset>

<fileset dir="${ext.lib.dir}">
<include name="*.jar" />
</fileset>

<fileset dir="${crypto.jar.dir}">
<include name="**/*.jar" />
</fileset>

<pathelement path="${dist.classes.dir}" />
</path>

<path id="test.classpath">
<fileset dir="${test.lib.dir}">
<include name="*.jar" />
</fileset>
</path>

<!-- set the build number based on environment variable, otherwise blank -->
<property environment="env" description="System environment variables (including those set by Hudson)" />

<target name="all" depends="clean, jars" description="Builds all jars" />

<target name="clean" description="Delete generated files.">
<echo message="Deleting generated files in dist" />
<delete dir="${dist.jar.dir}" />
<delete dir="${dist.classes.dir}" />
<delete dir="${dist.packages.dir}" />
<delete dir="${test.report.dir}" />
<delete>
<fileset file="${lib.dir}/azkaban-hadoopsecuritymanager-*.jar">
</fileset>
</delete>
</target>

<target name="build" depends="clean" description="Compile main source tree java files">
<delete>
<fileset file="${lib.dir}/azkaban-hadoopsecuritymanager-*.jar">
</fileset>
</delete>
<!-- copy the latest securitymanager jar -->
<copy todir="${lib.dir}">
<fileset file="${hadoopsecuritymanagerjar}">
</fileset>
</copy>

<delete dir="${dist.classes.dir}" />
<mkdir dir="${dist.classes.dir}" />

<!--copy todir="${dist.classes.dir}">
<property file="build.properties"/>
<property name="base.dir" value="${basedir}/../.."/>
<property name="dist.dir" value="${base.dir}/dist/jobtype"/>
<property name="lib.dir" value="${basedir}/lib"/>
<property name="ext.lib.dir" value="${base.dir}/extlib"/>
<property name="test.lib.dir" value="${base.dir}/test_lib"/>
<property name="test.resource.dir" value="${base.dir}/test_resource"/>
<property name="dist.jar.dir" value="${dist.dir}/jars"/>
<property name="dist.classes.dir" value="${dist.dir}/classes"/>
<property name="dist.classes.test.dir" value="${dist.dir}/testclasses"/>
<property name="dist.packages.dir" value="${dist.dir}/packages"/>
<property name="test.report.dir" value="${dist.dir}/report"/>
<property name="java.src.dir" value="${basedir}/src"/>
<property name="java.src.test.dir" value="${basedir}/test"/>
<property name="azkaban-jobtype-jar" value="${dist.jar.dir}/${name}-${version}.jar"/>
<property name="jobtypes.dir" value="${basedir}/jobtypes"/>
<property name="javatype.dir" value="${basedir}/jobtypes/java"/>
<property name="pigtype.dir" value="${basedir}/jobtypes/pig"/>
<property file="build.properties"/>
<property name="hadoopsecuritymanagerjar"
value="${base.dir}/dist/hadoopsecuritymanager/jars/azkaban-hadoopsecuritymanager-${version}.jar"/>
<property name="hadoopsecuritymanagerjar-yarn"
value="${base.dir}/dist/hadoopsecuritymanager-yarn/jars/azkaban-hadoopsecuritymanageryarn-${version}.jar"/>
<property name="hadoopsecuritymanager-common.java.src.dir"
value="${base.dir}/plugins/hadoopsecuritymanager-common/src"/>
<property name="crypto.jar.dir" value="${base.dir}/dist/crypto"/>
<property name="crypto.jar" value="${base.dir}/dist/crypto/jars/azkaban-crypto-${version}.jar"/>

<property name="version.file" value="${base.dir}/dist/package.version"/>
<property environment="env"/>

<path id="main.classpath">
<fileset dir="${lib.dir}">
<include name="*.jar"/>
</fileset>

<fileset dir="${ext.lib.dir}">
<include name="*.jar"/>
</fileset>

<fileset dir="${crypto.jar.dir}">
<include name="**/*.jar"/>
</fileset>

<pathelement path="${dist.classes.dir}"/>
</path>

<path id="test.classpath">
<fileset dir="${test.lib.dir}">
<include name="*.jar"/>
</fileset>
</path>

<!-- set the build number based on environment variable, otherwise blank -->
<property environment="env" description="System environment variables (including those set by Hudson)"/>

<target name="all" depends="clean, jars" description="Builds all jars"/>

<target name="clean" description="Delete generated files.">
<echo message="Deleting generated files in dist"/>
<delete dir="${dist.jar.dir}"/>
<delete dir="${dist.classes.dir}"/>
<delete dir="${dist.packages.dir}"/>
<delete dir="${test.report.dir}"/>
<delete>
<fileset file="${lib.dir}/azkaban-hadoopsecuritymanager-*.jar">
</fileset>
</delete>
</target>

<target name="build" depends="clean" description="Compile main source tree java files">
<delete>
<fileset file="${lib.dir}/azkaban-hadoopsecuritymanager-*.jar">
</fileset>
</delete>
<!-- copy the latest securitymanager jar -->
<copy todir="${lib.dir}">
<fileset file="${hadoopsecuritymanagerjar}">
</fileset>
</copy>

<delete dir="${dist.classes.dir}"/>
<mkdir dir="${dist.classes.dir}"/>

<!--copy todir="${dist.classes.dir}">
<fileset dir="${java.src.dir}">
<exclude name="**/*.java" />
</fileset>
</copy-->
<exclude name="**/*.java" />
</fileset>
</copy-->

<javac fork="true" destdir="${dist.classes.dir}"
target="1.8" debug="true" deprecation="false" failonerror="true">
<src path="${java.src.dir}" />
<src path="${hadoopsecuritymanager-common.java.src.dir}" />
<javac fork="true" destdir="${dist.classes.dir}"
target="1.8" debug="true" deprecation="false" failonerror="true">
<src path="${java.src.dir}"/>
<src path="${hadoopsecuritymanager-common.java.src.dir}"/>

<exclude name="**/examples/**"/>
<classpath refid="main.classpath" />
</javac>
<exclude name="**/examples/**"/>
<classpath refid="main.classpath"/>
</javac>


<delete dir="${dist.classes.test.dir}" />
<mkdir dir="${dist.classes.test.dir}" />
<delete dir="${dist.classes.test.dir}"/>
<mkdir dir="${dist.classes.test.dir}"/>
<javac fork="true" destdir="${dist.classes.test.dir}"
target="1.8" debug="true" deprecation="false" failonerror="true">
<src path="${java.src.test.dir}" />
<classpath refid="main.classpath" />
<classpath refid="test.classpath" />
<classpath>
<pathelement path="${test.resource.dir}" />
</classpath>
target="1.8" debug="true" deprecation="false" failonerror="true">
<src path="${java.src.test.dir}"/>
<classpath refid="main.classpath"/>
<classpath refid="test.classpath"/>
<classpath>
<pathelement path="${test.resource.dir}"/>
</classpath>
</javac>

</target>
</target>

<target name="junit" depends="build" description="running test cases">
<junit haltonfailure="no" failureproperty="test.failed" printsummary="yes">
<classpath>
<path refid="main.classpath" />
<path refid="test.classpath" />
<pathelement path="${dist.classes.dir}" />
<pathelement path="${dist.classes.test.dir}" />
<pathelement path="${test.resource.dir}" />
<path refid="main.classpath"/>
<path refid="test.classpath"/>
<pathelement path="${dist.classes.dir}"/>
<pathelement path="${dist.classes.test.dir}"/>
<pathelement path="${test.resource.dir}"/>

</classpath>

<batchtest fork="yes">
<fileset dir="${dist.classes.test.dir}" includes="**/*Test*.class" />
<fileset dir="${dist.classes.test.dir}" includes="**/*Test*.class"
excludes="**/TestHadoopSparkJob.class"/>
</batchtest>
</junit>
<fail message="Test failure detected, check test results." if="test.failed" />
<fail message="Test failure detected, check test results." if="test.failed"/>
</target>

<target name="jars" depends="junit" description="Compile main source tree java files">

<mkdir dir="${dist.jar.dir}" />
<jar destfile="${azkaban-jobtype-jar}">
<fileset dir="${dist.classes.dir}">
<include name="**/*.*" />
</fileset>
</jar>
</target>

<target name="package-jobtype" depends="jars" description="Creates a package for jobtypes">
<delete dir="${dist.packages.dir}" />
<mkdir dir="${dist.packages.dir}" />

<!-- Build jobtypes directory-->
<copy todir="${dist.packages.dir}" >
<fileset dir="${jobtypes.dir}" >
</fileset>
</copy>

<!-- Copy build version file -->
<copy file="${version.file}" todir="${dist.packages.dir}" />

<!-- Copy jobtype jar-->
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/java" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/hadoopJava" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.9.2" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.10.0" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.10.1" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.11.0" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.12.0" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/hive" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/hdfsToTeradata" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/teradataToHdfs" />
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/gobblin" />

<!-- Copy hadoopsecuritymanager-->
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/java" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/hadoopJava" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.9.2" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.10.0" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.10.1" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.11.0" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.12.0" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/hive" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/hdfsToTeradata" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/teradataToHdfs" />
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/gobblin" />

<!-- Copy hadoopsecuritymanager-yarn file -->
<copy file="${hadoopsecuritymanagerjar-yarn}" todir="${dist.packages.dir}" />

<!-- Copy crypto jar file -->
<copy todir="${dist.packages.dir}/hdfsToTeradata/lib" flatten="true" >
<fileset dir="${crypto.jar.dir}" excludes="**/log4j*.jar,**/commons-cli*.jar" includes="**/*.jar" />
</copy>
<copy todir="${dist.packages.dir}/teradataToHdfs/lib" flatten="true" >
<fileset dir="${crypto.jar.dir}" excludes="**/log4j*.jar,**/commons-cli*.jar" includes="**/*.jar" />
</copy>

<!-- Tarball it -->
<tar destfile="${dist.packages.dir}/${name}-${version}.tar.gz" compression="gzip" longfile="gnu">
<tarfileset dir="${dist.packages.dir}" prefix="${name}-${version}" filemode="755" />
</tar>
</target>

<target name="package" depends="package-jobtype" description="Create all packages">
</target>
<target name="jars" depends="junit" description="Compile main source tree java files">

<mkdir dir="${dist.jar.dir}"/>
<jar destfile="${azkaban-jobtype-jar}">
<fileset dir="${dist.classes.dir}">
<include name="**/*.*"/>
</fileset>
</jar>
</target>

<target name="package-jobtype" depends="jars" description="Creates a package for jobtypes">
<delete dir="${dist.packages.dir}"/>
<mkdir dir="${dist.packages.dir}"/>

<!-- Build jobtypes directory-->
<copy todir="${dist.packages.dir}">
<fileset dir="${jobtypes.dir}">
</fileset>
</copy>

<!-- Copy build version file -->
<copy file="${version.file}" todir="${dist.packages.dir}"/>

<!-- Copy jobtype jar-->
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/java"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/hadoopJava"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.9.2"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.10.0"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.10.1"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.11.0"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/pig-0.12.0"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/hive"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/hdfsToTeradata"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/teradataToHdfs"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/gobblin"/>
<copy file="${azkaban-jobtype-jar}" todir="${dist.packages.dir}/beelineHive"/>

<!-- Copy hadoopsecuritymanager-->
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/java"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/hadoopJava"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.9.2"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.10.0"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.10.1"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.11.0"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/pig-0.12.0"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/hive"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/hdfsToTeradata"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/teradataToHdfs"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/gobblin"/>
<copy file="${hadoopsecuritymanagerjar}" todir="${dist.packages.dir}/beelineHive"/>

<!-- Copy hadoopsecuritymanager-yarn file -->
<copy file="${hadoopsecuritymanagerjar-yarn}" todir="${dist.packages.dir}"/>

<!-- Copy crypto jar file -->
<copy todir="${dist.packages.dir}/hdfsToTeradata/lib" flatten="true">
<fileset dir="${crypto.jar.dir}" excludes="**/log4j*.jar,**/commons-cli*.jar" includes="**/*.jar"/>
</copy>
<copy todir="${dist.packages.dir}/teradataToHdfs/lib" flatten="true">
<fileset dir="${crypto.jar.dir}" excludes="**/log4j*.jar,**/commons-cli*.jar" includes="**/*.jar"/>
</copy>

<!-- Tarball it -->
<tar destfile="${dist.packages.dir}/${name}-${version}.tar.gz" compression="gzip" longfile="gnu">
<tarfileset dir="${dist.packages.dir}" prefix="${name}-${version}" filemode="755"/>
</tar>
</target>

<target name="package" depends="package-jobtype" description="Create all packages">
</target>

</project>
3 changes: 3 additions & 0 deletions plugins/jobtype/jobtypes/beelinehive/private.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
jobtype.class=azkaban.jobtype.BeelineHiveJob
hadoop.home=/usr/hdp/current
jobtype.classpath=${hadoop.home}/hadoop-client/hadoop-common.jar,${hadoop-home}/hadoop-client/lib/*,${hadoop.home}/hive-client/lib/*,/etc/hadoop/conf
Loading