feat: upgrade timeline service to 1.5

roles/yarn/apptimelineserver/tasks/install.yml

@@ -15,10 +15,25 @@
     group: root
     mode: "755"
 
+- name: Create YARN leveldb-timeline-store path
+  file:
+    path: "{{ yarn_site['yarn.timeline-service.leveldb-timeline-store.path'] }}"
+    state: directory
+    owner: "{{ yarn_user }}"
+    group: "{{ hadoop_group }}"
+    mode: "750"
+
 - name: Template YARN Timeline Server service file
   template:
     src: hadoop-yarn-timelineserver.service.j2
     dest: /usr/lib/systemd/system/hadoop-yarn-timelineserver.service
     owner: root
     group: root
     mode: "644"
+
+- name: Create a symbolic link to necessary tez jars
+  file:
+    src: "{{ item }}"
+    dest: "{{ hadoop_install_dir }}/share/hadoop/yarn/{{ item.split('/')[-1] }}"
+    state: link
+  loop: "{{ tez_jars }}"

roles/yarn/common/tasks/hdfs_init.yml

@@ -50,3 +50,14 @@
       owner: "{{ yarn_user }}"
       group: "{{ hadoop_group }}"
       mode: "1777"
+    - path: "{{ yarn_site['yarn.timeline-service.entity-group-fs-store.active-dir'] }}"
+      state: directory
+      owner: "{{ yarn_user }}"
+      group: "{{ hadoop_group }}"
+      mode: "1777"
+    - path: "{{ yarn_site['yarn.timeline-service.entity-group-fs-store.done-dir'] }}"
+      state: directory
+      owner: "{{ yarn_user }}"
+      group: "{{ hadoop_group }}"
+      mode: "0700"
+

tdp_vars_defaults/hadoop/hadoop.yml

@@ -113,6 +113,7 @@ auth_to_local:
     - RULE:[1:$1@$0]({{ hdfs_headless_principal }}@{{ realm }})s/.*/hdfs/
   yarn:
     - RULE:[2:$1/$2@$0]([rn]m/.*@{{ realm }})s/.*/yarn/
+    - RULE:[2:$1/$2@$0](ats/.*@{{ realm }})s/.*/yarn/
     - RULE:[1:$1@$0]({{ yarn_headless_principal }}@{{ realm }})s/.*/yarn/
   mapred:
     - RULE:[2:$1/$2@$0](jhs/.*@{{ realm }})s/.*/mapred/

tdp_vars_defaults/yarn/yarn.yml

@@ -12,7 +12,7 @@ hadoop_mapred_dir: /var/lib/mapred
 hadoop_mapred_pid_dir: /run/hadoop/mapred
 hadoop_mapred_log_dir: /var/log/hadoop/mapred
 
-# Propertiess
+# Properties
 hdfs_user: hdfs
 
 # TODO: make a yarn_site per service: rm, nm, ts
@@ -43,7 +43,6 @@ yarn_site:
   yarn.resourcemanager.webapp.spnego-principal: "HTTP/_HOST@{{ realm }}"
   yarn.resourcemanager.keytab: /etc/security/keytabs/rm.service.keytab
   yarn.resourcemanager.principal: "rm/_HOST@{{ realm }}"
-  yarn.resourcemanager.system-metrics-publisher.enabled: "true"
   yarn.resourcemanager.recovery.enabled: "true"
   yarn.resourcemanager.store.class: "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore"
   yarn.nodemanager.local-dirs: "{{ yarn_nodemanager_data_dirs }}"
@@ -88,14 +87,22 @@ yarn_site:
   yarn.timeline-service.client.best-effort: true
   yarn.timeline-service.client.max-retries: 3
   yarn.timeline-service.hostname: "{{ groups['yarn_ats'][0] | tosit.tdp.access_fqdn(hostvars) }}"
-  yarn.timeline-service.address: "0.0.0.0:{{ yarn_ats_rpc_port }}"
+  yarn.timeline-service.address: "{{ groups['yarn_ats'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ yarn_ats_rpc_port }}"
   yarn.timeline-service.webapp.https.address: "{{ groups['yarn_ats'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ yarn_ats_https_port }}"
   yarn.timeline-service.principal: ats/_HOST@{{ realm }}
   yarn.timeline-service.keytab: /etc/security/keytabs/ats.service.keytab
   # To enable Kerberos on the ATS UI
   yarn.timeline-service.http-authentication.type: kerberos
   yarn.timeline-service.http-authentication.kerberos.principal: HTTP/_HOST@{{ realm }}
   yarn.timeline-service.http-authentication.kerberos.keytab: /etc/security/keytabs/spnego.service.keytab
+  yarn.timeline-service.version: "1.5"
+  yarn.timeline-service.store-class: org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore
+  yarn.timeline-service.leveldb-timeline-store.path: "{{ hadoop_yarn_dir }}/ats/leveldb/"
+  yarn.timeline-service.entity-group-fs-store.active-dir: /ats/active
+  yarn.timeline-service.entity-group-fs-store.done-dir: /ats/done
+  yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes: org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl
+  yarn.timeline-service.entity-group-fs-store.summary-store: org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
+  yarn.system-metrics-publisher.enabled: "true"
   yarn.acl.enable: "true"
   yarn.admin.acl: yarn,knox
   yarn.log.server.url: "https://{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_https_port }}/jobhistory/logs"
@@ -163,3 +170,9 @@ mapred_jhs_restart: "no"
 # YARN NodeManagers decommission
 # List of NodeManagers to decommission. See https://docs.cloudera.com/HDPDocuments/HDP3/HDP-3.1.4/administration/content/decommissioning-slave-nodes.html
 yarn_nodemanagers_decommission: []
+
+# Tez JAR needed for YARN ATS 1.5. See https://docs.cloudera.com/HDPDocuments/HDP3/HDP-3.0.1/data-operating-system/content/upgrading_timeline_server_1_0_to_1_5.html
+tez_jars:
+  - /opt/tdp/tez/tez-yarn-timeline-cache-plugin-0.9.1-TDP-0.1.0-SNAPSHOT.jar
+  - /opt/tdp/tez/tez-api-0.9.1-TDP-0.1.0-SNAPSHOT.jar
+  - /opt/tdp/tez/tez-common-0.9.1-TDP-0.1.0-SNAPSHOT.jar