Goal
Asset Share Commons provides
Cart process for downloading assets. However, if the cart size is too big say 10-20GB, AEM might take a performance hit making it unusable for other activities. For creating such cart zips the following post throttles requests using
Sling Ordered queue,
Uploads the created carts to S3 (if AEM is configured with S3 data store, the same bucket is used for storing carts), creates
Presigned urls and
Email's users the download linkCarts created in S3 bucket are not deleted, assuming the
Datastore Garbage Collection task takes care of cleaning them up from data store during routine maintenance...
For creating S3 Presigned urls for individual assets
check this postPackage Install |
GithubBundle Whitelist For demo purposes i used
getAdministrativeResourceResolver(null) and not service resource resolver, so whitelist the bundle...
http://localhost:4502/system/console/configMgr/org.apache.sling.jcr.base.internal.LoginAdminWhitelistConfigure Limit The
direct download limit in the following screenshot was set to
50MB. Any carts
less then 50MB are directly downloaded from AEM
Carts
more than 50MB are uploaded to S3 and
Presigned url is emailed to user
http://localhost:4502/system/console/configMgr/apps.experienceaem.assets.EAEMS3ServiceDirect DownloadEmail Download LinkEmailSolution
1) Add an OSGI service
apps.experienceaem.assets.EAEMS3Service for creating
cart zips, uploading to S3, generate Presigned URLs with the following code...
package apps.experienceaem.assets;
import com.amazonaws.HttpMethod;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.day.cq.dam.api.Asset;
import com.day.cq.dam.commons.util.DamUtil;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.api.security.user.Authorizable;
import org.apache.jackrabbit.api.security.user.UserManager;
import org.apache.sling.api.request.RequestParameter;
import org.apache.sling.api.resource.Resource;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.api.resource.ValueMap;
import org.apache.sling.jcr.base.util.AccessControlUtil;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.metatype.annotations.AttributeDefinition;
import org.osgi.service.metatype.annotations.AttributeType;
import org.osgi.service.metatype.annotations.Designate;
import org.osgi.service.metatype.annotations.ObjectClassDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jcr.Node;
import javax.jcr.Session;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Component(
immediate=true ,
service={ EAEMS3Service.class }
)
@Designate(ocd = EAEMS3Service.Configuration.class)
public class EAEMS3Service {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
public static String ZIP_MIME_TYPE = "application/zip";
private static AmazonS3 s3Client = null;
private static TransferManager s3TransferManager = null;
private long cartFileS3Expiration = (1000 * 60 * 60);
private String s3BucketName = "";
private long directDownloadLimit = 52428800L; // 50 MB
@Activate
protected void activate(Configuration configuration) {
cartFileS3Expiration = configuration.cartFileS3Expiration();
s3BucketName = configuration.s3BucketName();
directDownloadLimit = configuration.directDownloadLimit();
logger.info("Creating s3Client and s3TransferManager...");
s3Client = AmazonS3ClientBuilder.defaultClient();
s3TransferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build();
}
public long getDirectDownloadLimit(){
return directDownloadLimit;
}
public String getS3PresignedUrl(String objectKey, String cartName, String mimeType){
String presignedUrl = "";
try{
if(StringUtils.isEmpty(objectKey)){
return presignedUrl;
}
ResponseHeaderOverrides nameHeader = new ResponseHeaderOverrides();
nameHeader.setContentType(mimeType);
nameHeader.setContentDisposition("attachment; filename=" + cartName);
GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3BucketName, objectKey)
.withMethod(HttpMethod.GET)
.withResponseHeaders(nameHeader)
.withExpiration(getCartFileExpirationDate());
URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
presignedUrl = url.toString();
logger.debug("Cart = " + cartName + ", S3 presigned url = " + presignedUrl);
}catch(Exception e){
logger.error("Error generating s3 presigned url for " + cartName);
}
return presignedUrl;
}
public String uploadToS3(String cartName, String cartTempFilePath) throws Exception{
File cartTempFile = new File(cartTempFilePath);
PutObjectRequest putRequest = new PutObjectRequest(s3BucketName, cartName, cartTempFile);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(ZIP_MIME_TYPE);
putRequest.setMetadata(metadata);
Upload upload = s3TransferManager.upload(putRequest);
upload.waitForCompletion();
if(!cartTempFile.delete()){
logger.warn("Error deleting temp cart from local file system after uploading to S3 - " + cartTempFilePath);
}
return cartName;
}
public String createTempZip(List<Asset> assets, String cartName) throws Exception{
File cartFile = File.createTempFile(cartName, ".tmp");
FileOutputStream cartFileStream = new FileOutputStream(cartFile);
ZipOutputStream zipStream = new ZipOutputStream( cartFileStream );
zipStream.setMethod(ZipOutputStream.DEFLATED);
zipStream.setLevel(Deflater.NO_COMPRESSION);
assets.forEach(asset -> {
BufferedInputStream inStream = new BufferedInputStream(asset.getOriginal().getStream());
try{
zipStream.putNextEntry(new ZipEntry(asset.getName()));
IOUtils.copyLarge(inStream, zipStream);
zipStream.closeEntry();
}catch(Exception e){
logger.error("Error adding zip entry - " + asset.getPath(), e);
}finally{
IOUtils.closeQuietly(inStream);
}
});
IOUtils.closeQuietly(zipStream);
return cartFile.getAbsolutePath();
}
public String getDirectDownloadUrl(List<Asset> assets){
StringBuilder directUrl = new StringBuilder();
directUrl.append("/content/dam/.assetdownload.zip/assets.zip?flatStructure=true&licenseCheck=false&");
for(Asset asset : assets){
directUrl.append("path=").append(asset.getPath()).append("&");
}
return directUrl.toString();
}
public List<Asset> getAssets(ResourceResolver resolver, String paths){
List<Asset> assets = new ArrayList<Asset>();
Resource assetResource = null;
for(String path : paths.split(",")){
assetResource = resolver.getResource(path);
if(assetResource == null){
continue;
}
assets.add(assetResource.adaptTo(Asset.class));
}
return assets;
}
public List<Asset> getAssets(ResourceResolver resolver, RequestParameter[] requestParameters){
List<Asset> assets = new ArrayList<Asset>();
if(ArrayUtils.isEmpty(requestParameters)){
return assets;
}
for (RequestParameter requestParameter : requestParameters) {
Resource resource = resolver.getResource(requestParameter.getString());
if(resource == null){
continue;
}
assets.add(resource.adaptTo(Asset.class));
}
return assets;
}
public long getSizeOfContents(List<Asset> assets) throws Exception{
long size = 0L;
Node node, metadataNode = null;
for(Asset asset : assets){
node = asset.adaptTo(Node.class);
metadataNode = node.getNode("jcr:content/metadata");
long bytes = Long.valueOf(DamUtil.getValue(metadataNode, "dam:size", "0"));
if (bytes == 0 && (asset.getOriginal() != null)) {
bytes = asset.getOriginal().getSize();
}
size = size + bytes;
}
return size;
}
public String getCartZipFileName(String username){
if(StringUtils.isEmpty(username)){
username = "anonymous";
}
String cartName = "cart-" + username;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
cartName = cartName + "-" + format.format(new Date()) + ".zip";
return cartName;
}
public Date getCartFileExpirationDate(){
Date expiration = new Date();
long expTimeMillis = expiration.getTime();
expTimeMillis = expTimeMillis + cartFileS3Expiration;
expiration.setTime(expTimeMillis);
return expiration;
}
public String getUserEmail(ResourceResolver resolver, String userId) throws Exception{
UserManager um = AccessControlUtil.getUserManager(resolver.adaptTo(Session.class));
Authorizable user = um.getAuthorizable(userId);
ValueMap profile = resolver.getResource(user.getPath() + "/profile").adaptTo(ValueMap.class);
return profile.get("email", "");
}
@ObjectClassDefinition(
name = "Experience AEM S3 for Download",
description = "Experience AEM S3 Presigned URLs for Downloading Asset Share Commons Carts"
)
public @interface Configuration {
@AttributeDefinition(
name = "Cart download S3 URL expiration",
description = "Cart download Presigned S3 URL expiration",
type = AttributeType.LONG
)
long cartFileS3Expiration() default (3 * 24 * 60 * 60 * 1000 );
@AttributeDefinition(
name = "Cart direct download limit",
description = "Cart size limit for direct download from AEM...",
type = AttributeType.LONG
)
long directDownloadLimit() default 52428800L; // 50MB
@AttributeDefinition(
name = "S3 Bucket Name e.g. eaem-s3-bucket",
description = "S3 Bucket Name e.g. eaem-s3-bucket",
type = AttributeType.STRING
)
String s3BucketName();
}
}
2) Add a servlet
apps.experienceaem.assets.EAEMS3DownloadServlet to process direct download and cart creation requests. Servlet check if the
pre zip size (of all assets combined) is less than a
configurable direct download limit directDownloadLimit; if size is less than the limit, the cart is available for immediate download from modal, otherwise an
email is sent to user when the
cart is processed in sling queue and ready for download...
package apps.experienceaem.assets;
import com.day.cq.dam.api.Asset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.api.security.user.Authorizable;
import org.apache.jackrabbit.api.security.user.UserManager;
import org.apache.sling.api.SlingHttpServletRequest;
import org.apache.sling.api.SlingHttpServletResponse;
import org.apache.sling.api.request.RequestParameter;
import org.apache.sling.api.resource.Resource;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.api.resource.ValueMap;
import org.apache.sling.api.servlets.SlingAllMethodsServlet;
import org.apache.sling.event.jobs.JobManager;
import org.apache.sling.jcr.base.util.AccessControlUtil;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jcr.Session;
import javax.servlet.Servlet;
import javax.servlet.ServletException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component(
service = Servlet.class,
property = {
"sling.servlet.methods=GET,POST",
"sling.servlet.paths=/bin/experience-aem/cart"
}
)
public class EAEMS3DownloadServlet extends SlingAllMethodsServlet {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final long GB_20 = 21474836480L;
@Reference
private EAEMS3Service eaems3Service;
@Reference
private JobManager jobManager;
public final void doPost(SlingHttpServletRequest request, SlingHttpServletResponse response)
throws ServletException, IOException {
doGet(request, response);
}
public final void doGet(SlingHttpServletRequest request, SlingHttpServletResponse response)
throws ServletException, IOException {
String paths = request.getParameter("paths");
if(StringUtils.isEmpty(paths)){
RequestParameter[] pathParams = request.getRequestParameters("path");
if(ArrayUtils.isEmpty(pathParams)){
response.sendError(403, "Missing path parameters");
return;
}
List<String> rPaths = new ArrayList<String>();
for(RequestParameter param : pathParams){
rPaths.add(param.getString());
}
paths = StringUtils.join(rPaths, ",");
}
logger.debug("Processing download of paths - " + paths);
ResourceResolver resolver = request.getResourceResolver();
List<Asset> assets = eaems3Service.getAssets(resolver, paths);
try{
long sizeOfContents = eaems3Service.getSizeOfContents(assets);
if(sizeOfContents > GB_20 ){
response.sendError(403, "Requested content too large");
return;
}
if(sizeOfContents < eaems3Service.getDirectDownloadLimit() ){
response.sendRedirect(eaems3Service.getDirectDownloadUrl(assets));
return;
}
String userId = request.getUserPrincipal().getName();
String email = eaems3Service.getUserEmail(resolver, userId);
if(StringUtils.isEmpty(email)){
response.sendError(500, "No email address registered for user - " + userId);
return;
}
String cartName = eaems3Service.getCartZipFileName(request.getUserPrincipal().getName());
logger.debug("Creating job for cart - " + cartName + ", with assets - " + paths);
Map<String, Object> payload = new HashMap<String, Object>();
payload.put(EAEMCartCreateJobConsumer.CART_NAME, cartName);
payload.put(EAEMCartCreateJobConsumer.ASSET_PATHS, paths);
payload.put(EAEMCartCreateJobConsumer.CART_RECEIVER_EMAIL, email);
jobManager.addJob(EAEMCartCreateJobConsumer.JOB_TOPIC, payload);
response.sendRedirect(request.getHeader("referer"));
}catch(Exception e){
logger.error("Error creating cart zip", e);
response.sendError(500, "Error creating cart zip - " + e.getMessage());
}
}
}
3) Add a sling job for processing carts in an ordered fashion
/apps/eaem-asc-s3-presigned-cart-urls/config/org.apache.sling.event.jobs.QueueConfiguration-eaem-cart.xml<?xml version="1.0" encoding="UTF-8"?>
<jcr:root xmlns:sling="http://sling.apache.org/jcr/sling/1.0" xmlns:jcr="http://www.jcp.org/jcr/1.0"
jcr:primaryType="sling:OsgiConfig"
queue.maxparallel="{Long}1"
queue.name="Experience AEM Cart Creation Queue"
queue.priority="MIN"
queue.retries="{Long}1"
queue.retrydelay="{Long}5000"
queue.topics="apps/experienceaem/assets/cart"
queue.type="ORDERED"/>
4) Create the email template
/apps/eaem-asc-s3-presigned-cart-urls/mail-templates/cart-template.htmlSubject: ${subject}
<table style="width:100%" width="100%" bgcolor="#ffffff" style="background-color:#ffffff;" border="0" cellpadding="0" cellspacing="0">
<tr>
<td style="width:100%"> </td>
</tr>
<tr>
<td style="width:100%">Assets in cart : ${assetNames}</td>
</tr>
<tr>
<td style="width:100%"> </td>
</tr>
<tr>
<td style="width:100%"><a href="${presignedUrl}">Click to download</a></td>
</tr>
<tr>
<td style="width:100%"> </td>
</tr>
<tr>
<td style="width:100%">Download link for copy paste in browser - ${presignedUrl}</a></td>
</tr>
</table>
5) Add a job consumer
apps.experienceaem.assets.EAEMCartCreateJobConsumer to process
cart requests put in queue and email user the
S3 presigned url link when its ready for download...
package apps.experienceaem.assets;
import com.day.cq.dam.api.Asset;
import com.day.cq.mailer.MessageGatewayService;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.api.resource.ResourceResolverFactory;
import org.apache.sling.event.jobs.Job;
import org.apache.sling.event.jobs.consumer.JobConsumer;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.day.cq.commons.mail.MailTemplate;
import com.day.cq.mailer.MessageGateway;
import org.apache.commons.lang.text.StrLookup;
import org.apache.commons.mail.Email;
import org.apache.commons.mail.HtmlEmail;
import javax.jcr.Session;
import javax.mail.internet.InternetAddress;
import java.util.*;
@Component(
immediate = true,
service = {JobConsumer.class},
property = {
"process.label = Experience AEM Cart Create Job Topic",
JobConsumer.PROPERTY_TOPICS + "=" + EAEMCartCreateJobConsumer.JOB_TOPIC
}
)
public class EAEMCartCreateJobConsumer implements JobConsumer {
private static final Logger log = LoggerFactory.getLogger(EAEMCartCreateJobConsumer.class);
public static final String JOB_TOPIC = "apps/experienceaem/assets/cart";
public static final String CART_NAME = "CART_NAME";
public static final String CART_RECEIVER_EMAIL = "CART_RECEIVER_EMAIL";
public static final String ASSET_PATHS = "ASSET_PATHS";
private static String EMAIL_TEMPLATE_PATH = "/apps/eaem-asc-s3-presigned-cart-urls/mail-templates/cart-template.html";
@Reference
private MessageGatewayService messageGatewayService;
@Reference
ResourceResolverFactory resourceResolverFactory;
@Reference
private EAEMS3Service eaems3Service;
@Override
public JobResult process(final Job job) {
long startTime = System.currentTimeMillis();
String cartName = (String)job.getProperty(CART_NAME);
String assetPaths = (String)job.getProperty(ASSET_PATHS);
String receiverEmail = (String)job.getProperty(CART_RECEIVER_EMAIL);
log.debug("Start processing cart - " + cartName);
ResourceResolver resolver = null;
try{
resolver = resourceResolverFactory.getAdministrativeResourceResolver(null);
List<Asset> assets = eaems3Service.getAssets(resolver, assetPaths);
String cartTempFilePath = eaems3Service.createTempZip(assets, cartName);
log.debug("Cart - " + cartName + ", creation took " + ((System.currentTimeMillis() - startTime) / 1000) + " secs");
String objectKey = eaems3Service.uploadToS3(cartName, cartTempFilePath);
String presignedUrl = eaems3Service.getS3PresignedUrl(objectKey, cartName, EAEMS3Service.ZIP_MIME_TYPE);
log.debug("Cart - " + cartName + ", with object key - " + objectKey + ", creation and upload to S3 took " + ((System.currentTimeMillis() - startTime) / 1000) + " secs");
List<String> assetNames = new ArrayList<String>();
for(Asset asset : assets){
assetNames.add(asset.getName());
}
log.debug("Sending email to - " + receiverEmail + ", with assetNames in cart - " + cartName + " - " + StringUtils.join(assetNames, ","));
Map<String, String> emailParams = new HashMap<String,String>();
emailParams.put("subject", "Ready for download - " + cartName);
emailParams.put("assetNames", StringUtils.join(assetNames, ","));
emailParams.put("presignedUrl", presignedUrl);
sendMail(resolver, emailParams, receiverEmail);
log.debug("End processing cart - " + cartName);
}catch(Exception e){
log.error("Error creating cart - " + cartName + ", with assets - " + assetPaths, e);
return JobResult.FAILED;
}finally{
if(resolver != null){
resolver.close();
}
}
return JobResult.OK;
}
private Email sendMail(ResourceResolver resolver, Map<String, String> emailParams, String recipientEmail) throws Exception{
MailTemplate mailTemplate = MailTemplate.create(EMAIL_TEMPLATE_PATH, resolver.adaptTo(Session.class));
if (mailTemplate == null) {
throw new Exception("Template missing - " + EMAIL_TEMPLATE_PATH);
}
Email email = mailTemplate.getEmail(StrLookup.mapLookup(emailParams), HtmlEmail.class);
email.setTo(Collections.singleton(new InternetAddress(recipientEmail)));
MessageGateway<Email> messageGateway = messageGatewayService.getGateway(email.getClass());
messageGateway.send(email);
return email;
}
}
6) Create a component /
apps/eaem-asc-s3-presigned-cart-urls/components/download with
sling:resourceSuperType /apps/asset-share-commons/components/modals/download to provide the
Email download link functionality
7) Create a sling model
apps.experienceaem.assets.EAEMDownload for use in the download HTL script
package apps.experienceaem.assets;
import org.apache.sling.api.SlingHttpServletRequest;
import org.apache.sling.models.annotations.Model;
import org.apache.sling.models.annotations.Required;
import org.apache.sling.models.annotations.injectorspecific.OSGiService;
import org.apache.sling.models.annotations.injectorspecific.Self;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.PostConstruct;
@Model(
adaptables = {SlingHttpServletRequest.class},
resourceType = {EAEMDownload.RESOURCE_TYPE}
)
public class EAEMDownload {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
protected static final String RESOURCE_TYPE = "/apps/eaem-asc-s3-presigned-cart-urls/components/download";
@Self
@Required
protected SlingHttpServletRequest request;
@OSGiService
@Required
private EAEMS3Service eaems3Service;
protected Long directDownloadLimit;
protected Long cartSize;
@PostConstruct
protected void init() {
directDownloadLimit = eaems3Service.getDirectDownloadLimit();
try{
cartSize = eaems3Service.getSizeOfContents(eaems3Service.getAssets(request.getResourceResolver(),
request.getRequestParameters("path")));
}catch (Exception e){
logger.error("Error calculating cart size", e);
}
}
public long getDirectDownloadLimit() {
return this.directDownloadLimit;
}
public long getCartSize() {
return this.cartSize;
}
}
Add the sling model package in
eaem-asc-s3-presigned-cart-urls\bundle\pom.xml<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<extensions>true</extensions>
<configuration>
<instructions>
<Bundle-SymbolicName>apps.experienceaem.assets.eaem-asc-s3-presigned-cart-urls-bundle</Bundle-SymbolicName>
<Sling-Model-Packages>
apps.experienceaem.assets
</Sling-Model-Packages>
</instructions>
</configuration>
</plugin>
8) Add the necessary changes to modal code
/apps/eaem-asc-s3-presigned-cart-urls/components/download/download.html and add component in
action page eg.
http://localhost:4502/editor.html/content/asset-share-commons/en/light/actions/download.html<sly data-sly-use.eaemDownload="apps.experienceaem.assets.EAEMDownload"></sly>
<form method="post"
action="/bin/experience-aem/cart"
target="download"
data-asset-share-id="download-modal"
class="ui modal cmp-modal-download--wrapper cmp-modal">
.......................
<div data-sly-test.isCartDownload="${eaemDownload.cartSize > eaemDownload.directDownloadLimit}"
class="ui attached warning message cmp-message">
<span class="detail">Size exceeds limit for direct download, you'll receive an email when the cart is ready for download</span>
</div>
.......................
<button type="submit" class="ui positive primary right labeled icon button ${isMaxSize ? 'disabled': ''}">
${isCartDownload ? 'Email when ready' : properties['downloadButton'] }
<i class="download icon"></i>
</button>