-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcmgcloud.py
executable file
·202 lines (168 loc) · 6.75 KB
/
cmgcloud.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
#! /usr/bin/env python
"""Manages a Cloudera Hadoop Cluster start and stop process.
"""
# sudo pip install cm-api
# sudo pip install --upgrade google-api-python-client
import logging
import sys
from optparse import OptionParser
# Get a handle to the API client
from cm_api.api_client import ApiResource
# Get a handle to the oAuth2 Api
from oauth2client.client import GoogleCredentials
from googleapiclient import discovery
from time import sleep
__author__ = "Sergio Rodriguez"
__license__ = "GPL"
__version__ = "0.1.0"
__maintainer__ = "Sergio Rodriguez"
__email__ = "[email protected]"
__status__ = "Development"
parser = OptionParser()
parser.add_option('-H', '--host', type='string', dest='cm_host', help='The Cloudera Manager host i.e: 93.44.55.66')
parser.add_option('-U', '--user', type='string', dest='username', default='admin', help='Admin user name')
parser.add_option('-P', '--password', type='string', dest='password', default='admin', help='Password of the admin user name')
parser.add_option('-O', '--operation', type='string', dest='operation', help='Operation to perform on the cluster: start or stop')
parser.add_option('-G', '--google-cloud', dest='switch_gc_instances', default=False, help='Start/Stop Google Cloud instances aswell', action='store_true')
parser.add_option('-J', '--project', type='string', dest='gcproject', help='Specify Google Cloud Project')
parser.add_option('-Z', '--zone', type='string', dest='gczone', help='Specify Google Cloud Zone')
parser.add_option('-v', '--verbose', dest='verbose', default=False, help='Enable Verbose Output.', action='store_true')
(options, args) = parser.parse_args()
if options.cm_host is None:
logging.error('Please specify a CM host with -H or --host.')
parser.print_help()
sys.exit(-1)
if options.operation is None:
logging.error('Please specify an operation with -O start/stop or --operation start/stop.')
parser.print_help()
sys.exit(-1)
if options.switch_gc_instances is None or options.gcproject is None:
logging.error('Please specify Google Cloud Project with -J or --project.')
parser.print_help()
sys.exit(-1)
if options.switch_gc_instances is None or options.gczone is None:
logging.error('Please specify Google Cloud Zone with -Z or --zone.')
parser.print_help()
sys.exit(-1)
if options.verbose:
logging.basicConfig(
level=logging.INFO,
)
cm_host = options.cm_host
api = ApiResource(cm_host, username=options.username, password=options.password)
def shutdown_cluster():
# Get a list of all clusters
cdh5 = None
for c in api.get_all_clusters():
logging.info('Found cluster: ' + c.name)
if c.version == "CDH5":
cdh5 = c
for s in cdh5.get_all_services():
logging.info(s)
logging.info("Now stopping " + c.name)
cmd = c.stop()
print "Stopping the cluster. This might take a while."
while cmd.success == None and len(c.get_commands()) > 0:
sleep(5)
logging.info("Pending jobs: " + str(len(c.get_commands())))
cmd = cmd.fetch()
if cmd.success != True:
print "Cluster " + c.name + " stop failed: " + cmd.resultMessage
exit(0)
print "Cluster " + c.name + " stop succeeded"
#if s.type == "HDFS":
# hdfs = s
# print logging.info(hdfs.name + ":" + hdfs.serviceState + ":" + hdfs.healthSummary)
def start_cluster():
# Get a list of all clusters
cdh5 = None
for c in api.get_all_clusters():
logging.info('Found cluster: ' + c.name)
if c.version == "CDH5":
cdh5 = c
for s in cdh5.get_all_services():
logging.info(s)
logging.info("Now starting " + c.name)
cmd = c.start()
print "Starting the cluster. This might take a while."
while cmd.success == None and len(c.get_commands()) > 0:
sleep(5)
logging.info("Pending jobs: " + str(len(c.get_commands())))
cmd = cmd.fetch()
if cmd.success != True:
print "Cluster " + c.name + " start failed: " + cmd.resultMessage
exit(0)
print "Cluster " + c.name + " start succeeded"
def get_google_credentials():
credentials = GoogleCredentials.get_application_default()
compute = discovery.build('compute', 'v1', credentials=credentials)
# [START list_instances]
def list_instances(compute, project, zone):
result = compute.instances().list(project=project, zone=zone).execute()
return result['items']
# [END list_instances]
# [START start_instance]
def start_instance(compute, project, zone, name):
return compute.instances().start(
project=project,
zone=zone,
instance=name).execute()
# [END start_instance]
# [START stop_instance]
def stop_instance(compute, project, zone, name):
return compute.instances().stop(
project=project,
zone=zone,
instance=name).execute()
# [END stop_instance]
# [START wait_for_operation]
def wait_for_operation(compute, project, zone, operation):
print('Waiting for operation to finish...')
while True:
result = compute.zoneOperations().get(
project=project,
zone=zone,
operation=operation).execute()
if result['status'] == 'DONE':
print("done.")
if 'error' in result:
raise Exception(result['error'])
return result
sleep(1)
# [END wait_for_operation]
# [START start_gc_cluster]
def start_gc_cluster():
credentials = GoogleCredentials.get_application_default()
compute = discovery.build('compute', 'v1', credentials=credentials)
instances = list_instances(compute, options.gcproject, options.gczone)
print('Instances in project %s and zone %s:' % (options.gcproject, options.gczone))
for instance in instances:
print(' - ' + instance['name'] + ': Starting instance...')
operation = start_instance(compute, options.gcproject, options.gczone, instance['name'])
wait_for_operation(compute, options.gcproject, options.gczone, operation['name'])
# [END start_gc_cluster]
# [START stop_gc_cluster]
def stop_gc_cluster():
credentials = GoogleCredentials.get_application_default()
compute = discovery.build('compute', 'v1', credentials=credentials)
instances = list_instances(compute, options.gcproject, options.gczone)
print('Instances in project %s and zone %s:' % (options.gcproject, options.gczone))
for instance in instances:
print(' - ' + instance['name'] + ': Stopping instance...')
operation = stop_instance(compute, options.gcproject, options.gczone, instance['name'])
wait_for_operation(compute, options.gcproject, options.gczone, operation['name'])
# [END stop_gc_cluster]
def main():
#wait_for_operation(compute, project, zone, operation['name'])
if options.operation == "start":
if options.switch_gc_instances:
start_gc_cluster()
# Should wait something for CM to start as a service or check open port
sleep(60)
start_cluster()
if options.operation == "stop":
shutdown_cluster()
if options.switch_gc_instances:
stop_gc_cluster()
if __name__ == "__main__":
main()