Python script to extract source using sp-config

Identity Security Cloud (ISC) ISC Discussion and Questions
1

I created a simple python script to extract the Sources, Transforms, Identity_profiles and Connector Rules. Basically it gets a list of the objects, and then exports the individual files info separate files for each object.
It reformats the files so they are a little more readable, rather than just one long line.

Save it as a .py file(sample.py for example), change the <>, <<CLIENT_ID>> and <<CLIENT_SECRET>> to the values for your environment. To run it, just enter:
python3 sample
It creates the necessary directories for the 4 types of objects.

import requests
import json
import os
from time import sleep

token_url = "https://<<TENANT>>.api.identitynow.com/oauth/token"
#client (application) credentials 
client_id = '<<CLIENT_ID>>'
client_secret = '<<CLIENT_SECRET>>'

data = {'grant_type': 'client_credentials'}

access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(client_id, client_secret))

tokens = json.loads(access_token_response.text)

headers = {'Content-Type': 'application/json','Authorization': 'Bearer ' + tokens['access_token']}

baseUrl="https://<<TENANT>>.api.identitynow.com"

# Start export, return jobID
def process_config(type, object):
	#print("getting config for ",object, ": Type:",type)
	exportConfigURL = baseUrl+"/beta/sp-config/export"
	payload = json.dumps({
	  "description": "Automated Export-"+type+"-"+object,
	  "excludeTypes": [],
	  "includeTypes": [
	    type
	  ],
	  "objectOptions": {
	    type: {
	      "includedIds": [],
	      "includedNames": [
	        object
	      ]
	    }
	  }
	})
	response = requests.request("POST", exportConfigURL, headers=headers, data=payload)
	#print(response.text)
	json_data=json.loads(response.text)	
	return json_data['jobId']

# Get Source code
def getSource(jobId):
	print("getting source for jobID:", jobId)
	sourceURL = baseUrl+"/beta/sp-config/export/"+jobId+"/download"
	payload={}
	response = requests.request("GET", sourceURL, headers=headers, data=payload)
	return response.text

def getObjects(sourceType, listURL):
# $$$$$$$ get list of sources
	listSourcesURL = listURL
	payload={}
	response = requests.request("GET", listSourcesURL, headers=headers, data=payload)
	json_array= json.loads(response.text)
	for source in json_array:
		print("Getting config for:", source['name'],  ":", source['id'])
		jobId=process_config(sourceType, source['name'])

		# return exported source
		sleep(5)
		sourceCode=getSource(jobId)
		if (os.path.isdir(sourceType)==False):
			os.mkdir(sourceType)
		f = open(sourceType+"/1"+source['name']+".json", "w")
		f.write(sourceCode)
		f.close()
	
	# Reformat source file
		with open(sourceType+"/1"+source['name']+".json") as read_file:
			datafile = json.load(read_file)
			with open(sourceType+"/"+source['name']+".json", "w") as write_file:
				json.dump(datafile, write_file, indent=4, sort_keys=True)
			os.remove(sourceType+"/1"+source['name']+".json")


# $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ Main process
print("##########   Getting Source             ##########")
getObjects("SOURCE", baseUrl+"/beta/sources?count=true")
print("##########   Getting Transforms         ##########")
getObjects("TRANSFORM", baseUrl+"/beta/transforms?limit=250&count=true&filters=internal eq false")
print("##########   Getting Identity Profile   ##########")
getObjects("IDENTITY_PROFILE", baseUrl+"/beta/identity-profiles?count=true")
print("##########   Getting Connector Rules   ##########")
getObjects("RULE", baseUrl+"/beta/connector-rules?count=true")
#print("##########   Getting All Rules         ##########")
# This needs a different getting of the name 
#  The server objects cannot be retreived this way. Needs further investigation
#getObjects("RULE", baseUrl+"/cc/api/rule/list")
3 Likes
2

This is amazing! Thank you for this @chrisp.